diff --git a/.gitignore b/.gitignore index e63284190..6761d7ea1 100644 --- a/.gitignore +++ b/.gitignore @@ -82,4 +82,5 @@ tags # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode #IDE (GoLand) specific .idea/ - +# test-environment files +registry-login.conf diff --git a/api/v1/helper.go b/api/v1/helper.go index ba2429caf..664531260 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -20,7 +20,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) const ( @@ -47,6 +49,8 @@ var log = logf.Log.WithName("sriovnetwork") // Vendor ID, Physical Function Device ID, Virtual Function Device ID var NicIDMap = []string{} +var InitialState SriovNetworkNodeState + // NetFilterType Represents the NetFilter tags to be used type NetFilterType int @@ -211,6 +215,80 @@ func GetVfDeviceID(deviceID string) string { return "" } +func IsSwitchdevModeSpec(spec SriovNetworkNodeStateSpec) bool { + for _, iface := range spec.Interfaces { + if iface.EswitchMode == ESwithModeSwitchDev { + return true + } + } + return false +} + +func FindInterface(interfaces Interfaces, name string) (iface Interface, err error) { + for _, i := range interfaces { + if i.Name == name { + return i, nil + } + } + return Interface{}, fmt.Errorf("unable to find interface: %v", name) +} + +func NeedToUpdateSriov(ifaceSpec *Interface, ifaceStatus *InterfaceExt) bool { + if ifaceSpec.Mtu > 0 { + mtu := ifaceSpec.Mtu + if mtu != ifaceStatus.Mtu { + log.V(2).Info("NeedToUpdateSriov(): MTU needs update", "desired", mtu, "current", ifaceStatus.Mtu) + return true + } + } + + if ifaceSpec.NumVfs != ifaceStatus.NumVfs { + log.V(2).Info("NeedToUpdateSriov(): NumVfs needs update", "desired", ifaceSpec.NumVfs, "current", ifaceStatus.NumVfs) + return true + } + if ifaceSpec.NumVfs > 0 { + for _, vfStatus := range ifaceStatus.VFs { + ingroup := false + for _, groupSpec := range ifaceSpec.VfGroups { + if IndexInRange(vfStatus.VfID, groupSpec.VfRange) { + ingroup = true + if groupSpec.DeviceType != consts.DeviceTypeNetDevice { + if groupSpec.DeviceType != vfStatus.Driver { + log.V(2).Info("NeedToUpdateSriov(): Driver needs update", + "desired", groupSpec.DeviceType, "current", vfStatus.Driver) + return true + } + } else { + if StringInArray(vfStatus.Driver, vars.DpdkDrivers) { + log.V(2).Info("NeedToUpdateSriov(): Driver needs update", + "desired", groupSpec.DeviceType, "current", vfStatus.Driver) + return true + } + if vfStatus.Mtu != 0 && groupSpec.Mtu != 0 && vfStatus.Mtu != groupSpec.Mtu { + log.V(2).Info("NeedToUpdateSriov(): VF MTU needs update", + "vf", vfStatus.VfID, "desired", groupSpec.Mtu, "current", vfStatus.Mtu) + return true + } + + // this is needed to be sure the admin mac address is configured as expected + if ifaceSpec.ExternallyManaged { + log.V(2).Info("NeedToUpdateSriov(): need to update the device as it's externally manage", + "device", ifaceStatus.PciAddress) + return true + } + } + break + } + } + if !ingroup && StringInArray(vfStatus.Driver, vars.DpdkDrivers) { + // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver. + return true + } + } + } + return false +} + type ByPriority []SriovNetworkNodePolicy func (a ByPriority) Len() int { diff --git a/cmd/sriov-network-config-daemon/service.go b/cmd/sriov-network-config-daemon/service.go index fc9cfc6d9..3305d7e8e 100644 --- a/cmd/sriov-network-config-daemon/service.go +++ b/cmd/sriov-network-config-daemon/service.go @@ -24,15 +24,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/virtual" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/version" - - snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" ) var ( @@ -57,11 +58,16 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { setupLog.V(2).Info("sriov-config-service", "version", version.Version) setupLog.V(0).Info("Starting sriov-config-service") + + // Mark that we are running on host + vars.UsingSystemdMode = true + vars.InChroot = true + supportedNicIds, err := systemd.ReadSriovSupportedNics() if err != nil { setupLog.Error(err, "failed to read list of supported nic ids") sriovResult := &systemd.SriovResult{ - SyncStatus: "Failed", + SyncStatus: consts.SyncStatusFailed, LastSyncError: fmt.Sprintf("failed to read list of supported nic ids: %v", err), } err = systemd.WriteSriovResult(sriovResult) @@ -78,7 +84,7 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { if _, err := os.Stat(systemd.SriovSystemdConfigPath); !errors.Is(err, os.ErrNotExist) { setupLog.Error(err, "failed to read the sriov configuration file", "path", systemd.SriovSystemdConfigPath) sriovResult := &systemd.SriovResult{ - SyncStatus: "Failed", + SyncStatus: consts.SyncStatusFailed, LastSyncError: fmt.Sprintf("failed to read the sriov configuration file in path %s: %v", systemd.SriovSystemdConfigPath, err), } err = systemd.WriteSriovResult(sriovResult) @@ -91,67 +97,64 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { nodeStateSpec = &systemd.SriovConfig{ Spec: sriovv1.SriovNetworkNodeStateSpec{}, UnsupportedNics: false, - PlatformType: utils.Baremetal, + PlatformType: consts.Baremetal, } } setupLog.V(2).Info("sriov-config-service", "config", nodeStateSpec) - - storeManager, err := utils.NewStoreManager(true) + hostHelpers, err := helper.NewDefaultHostHelpers() if err != nil { - setupLog.Error(err, "failed to create store manager") - return err + setupLog.Error(err, "failed to create hostHelpers") + return updateSriovResultErr("failed to create hostHelpers") } - // Load kernel modules - hostManager := host.NewHostManager(true) - _, err = hostManager.TryEnableRdma() + platformHelper, err := platforms.NewDefaultPlatformHelper() + if err != nil { + setupLog.Error(err, "failed to create platformHelpers") + return updateSriovResultErr("failed to create platformHelpers") + } + + _, err = hostHelpers.TryEnableRdma() if err != nil { setupLog.Error(err, "warning, failed to enable RDMA") } - hostManager.TryEnableTun() - hostManager.TryEnableVhostNet() + hostHelpers.TryEnableTun() + hostHelpers.TryEnableVhostNet() var configPlugin plugin.VendorPlugin var ifaceStatuses []sriovv1.InterfaceExt - if nodeStateSpec.PlatformType == utils.Baremetal { + if nodeStateSpec.PlatformType == consts.Baremetal { // Bare metal support - ifaceStatuses, err = utils.DiscoverSriovDevices(nodeStateSpec.UnsupportedNics, storeManager) + vars.DevMode = nodeStateSpec.UnsupportedNics + ifaceStatuses, err = hostHelpers.DiscoverSriovDevices(hostHelpers) if err != nil { setupLog.Error(err, "failed to discover sriov devices on the host") - return fmt.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service: failed to discover sriov devices on the host: %v", err)) } // Create the generic plugin - configPlugin, err = generic.NewGenericPlugin(true, hostManager, storeManager) + configPlugin, err = generic.NewGenericPlugin(hostHelpers) if err != nil { setupLog.Error(err, "failed to create generic plugin") - return fmt.Errorf("sriov-config-service failed to create generic plugin %v", err) - } - } else if nodeStateSpec.PlatformType == utils.VirtualOpenStack { - // Openstack support - metaData, networkData, err := utils.GetOpenstackData(false) - if err != nil { - setupLog.Error(err, "failed to read OpenStack data") - return fmt.Errorf("sriov-config-service failed to read OpenStack data: %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service failed to create generic plugin %v", err)) } - - openStackDevicesInfo, err := utils.CreateOpenstackDevicesInfo(metaData, networkData) + } else if nodeStateSpec.PlatformType == consts.VirtualOpenStack { + err = platformHelper.CreateOpenstackDevicesInfo() if err != nil { setupLog.Error(err, "failed to read OpenStack data") - return fmt.Errorf("sriov-config-service failed to read OpenStack data: %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service failed to read OpenStack data: %v", err)) } - ifaceStatuses, err = utils.DiscoverSriovDevicesVirtual(openStackDevicesInfo) + ifaceStatuses, err = platformHelper.DiscoverSriovDevicesVirtual() if err != nil { setupLog.Error(err, "failed to read OpenStack data") - return fmt.Errorf("sriov-config-service: failed to read OpenStack data: %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service: failed to read OpenStack data: %v", err)) } // Create the virtual plugin - configPlugin, err = virtual.NewVirtualPlugin(true) + configPlugin, err = virtual.NewVirtualPlugin(hostHelpers) if err != nil { setupLog.Error(err, "failed to create virtual plugin") - return fmt.Errorf("sriov-config-service: failed to create virtual plugin %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service: failed to create virtual plugin %v", err)) } } @@ -163,18 +166,18 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { _, _, err = configPlugin.OnNodeStateChange(nodeState) if err != nil { setupLog.Error(err, "failed to run OnNodeStateChange to update the generic plugin status") - return fmt.Errorf("sriov-config-service: failed to run OnNodeStateChange to update the generic plugin status %v", err) + return updateSriovResultErr(fmt.Sprintf("sriov-config-service: failed to run OnNodeStateChange to update the generic plugin status %v", err)) } sriovResult := &systemd.SriovResult{ - SyncStatus: "Succeeded", + SyncStatus: consts.SyncStatusSucceeded, LastSyncError: "", } err = configPlugin.Apply() if err != nil { setupLog.Error(err, "failed to run apply node configuration") - sriovResult.SyncStatus = "Failed" + sriovResult.SyncStatus = consts.SyncStatusFailed sriovResult.LastSyncError = err.Error() } @@ -187,3 +190,17 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { setupLog.V(0).Info("shutting down sriov-config-service") return nil } + +func updateSriovResultErr(errMsg string) error { + sriovResult := &systemd.SriovResult{ + SyncStatus: consts.SyncStatusFailed, + LastSyncError: errMsg, + } + + err := systemd.WriteSriovResult(sriovResult) + if err != nil { + log.Log.Error(err, "failed to write sriov result file", "content", *sriovResult) + return fmt.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + } + return nil +} diff --git a/cmd/sriov-network-config-daemon/start.go b/cmd/sriov-network-config-daemon/start.go index a7feffbde..1d7d4d00c 100644 --- a/cmd/sriov-network-config-daemon/start.go +++ b/cmd/sriov-network-config-daemon/start.go @@ -24,8 +24,6 @@ import ( "strings" "time" - configv1 "github.com/openshift/api/config/v1" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/spf13/cobra" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -35,12 +33,17 @@ import ( "k8s.io/client-go/util/connrotation" "sigs.k8s.io/controller-runtime/pkg/log" + configv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/daemon" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/version" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) var ( @@ -70,8 +73,11 @@ func runStartCmd(cmd *cobra.Command, args []string) error { snolog.InitLog() setupLog := log.Log.WithName("sriov-network-config-daemon") - // To help debugging, immediately log version - setupLog.V(2).Info("sriov-network-config-daemon", "version", version.Version) + // Mark that we are running inside a container + vars.UsingSystemdMode = false + if startOpts.systemd { + vars.UsingSystemdMode = true + } if startOpts.nodeName == "" { name, ok := os.LookupEnv("NODE_NAME") @@ -80,6 +86,7 @@ func runStartCmd(cmd *cobra.Command, args []string) error { } startOpts.nodeName = name } + vars.NodeName = startOpts.nodeName // This channel is used to ensure all spawned goroutines exit when we exit. stopCh := make(chan struct{}) @@ -102,7 +109,9 @@ func runStartCmd(cmd *cobra.Command, args []string) error { var config *rest.Config var err error - if os.Getenv("CLUSTER_TYPE") == utils.ClusterTypeOpenshift { + // On openshift we use the kubeconfig from kubelet on the node where the daemon is running + // this allow us to improve security as every daemon has access only to its own node + if vars.ClusterType == consts.ClusterTypeOpenshift { kubeconfig, err := clientcmd.LoadFromFile("/host/etc/kubernetes/kubeconfig") if err != nil { setupLog.Error(err, "failed to load kubelet kubeconfig") @@ -110,7 +119,7 @@ func runStartCmd(cmd *cobra.Command, args []string) error { clusterName := kubeconfig.Contexts[kubeconfig.CurrentContext].Cluster apiURL := kubeconfig.Clusters[clusterName].Server - url, err := url.Parse(apiURL) + urlPath, err := url.Parse(apiURL) if err != nil { setupLog.Error(err, "failed to parse api url from kubelet kubeconfig") } @@ -118,8 +127,14 @@ func runStartCmd(cmd *cobra.Command, args []string) error { // The kubernetes in-cluster functions don't let you override the apiserver // directly; gotta "pass" it via environment vars. setupLog.V(0).Info("overriding kubernetes api", "new-url", apiURL) - os.Setenv("KUBERNETES_SERVICE_HOST", url.Hostname()) - os.Setenv("KUBERNETES_SERVICE_PORT", url.Port()) + err = os.Setenv("KUBERNETES_SERVICE_HOST", urlPath.Hostname()) + if err != nil { + setupLog.Error(err, "failed to set KUBERNETES_SERVICE_HOST environment variable") + } + err = os.Setenv("KUBERNETES_SERVICE_PORT", urlPath.Port()) + if err != nil { + setupLog.Error(err, "failed to set KUBERNETES_SERVICE_PORT environment variable") + } } kubeconfig := os.Getenv("KUBECONFIG") @@ -134,57 +149,72 @@ func runStartCmd(cmd *cobra.Command, args []string) error { return err } + vars.Config = config + vars.Scheme = scheme.Scheme + closeAllConns, err := updateDialer(config) if err != nil { return err } - sriovnetworkv1.AddToScheme(scheme.Scheme) - mcfgv1.AddToScheme(scheme.Scheme) - configv1.Install(scheme.Scheme) + err = sriovnetworkv1.AddToScheme(scheme.Scheme) + if err != nil { + setupLog.Error(err, "failed to load sriov network CRDs to scheme") + return err + } + + err = mcfgv1.AddToScheme(scheme.Scheme) + if err != nil { + setupLog.Error(err, "failed to load machine config CRDs to scheme") + return err + } + + err = configv1.Install(scheme.Scheme) + if err != nil { + setupLog.Error(err, "failed to load openshift config CRDs to scheme") + return err + } snclient := snclientset.NewForConfigOrDie(config) kubeclient := kubernetes.NewForConfigOrDie(config) - openshiftContext, err := utils.NewOpenshiftContext(config, scheme.Scheme) + + hostHelpers, err := helper.NewDefaultHostHelpers() + if err != nil { + setupLog.Error(err, "failed to create hostHelpers") + return err + } + + platformHelper, err := platforms.NewDefaultPlatformHelper() if err != nil { + setupLog.Error(err, "failed to create platformHelper") return err } config.Timeout = 5 * time.Second writerclient := snclientset.NewForConfigOrDie(config) - mode := os.Getenv("DEV_MODE") - devMode := false - if mode == "TRUE" { - devMode = true - setupLog.V(0).Info("dev mode enabled") - } - - eventRecorder := daemon.NewEventRecorder(writerclient, startOpts.nodeName, kubeclient) + eventRecorder := daemon.NewEventRecorder(writerclient, kubeclient) defer eventRecorder.Shutdown() setupLog.V(0).Info("starting node writer") - nodeWriter := daemon.NewNodeStateStatusWriter(writerclient, startOpts.nodeName, closeAllConns, eventRecorder, devMode) - - destdir := os.Getenv("DEST_DIR") - if destdir == "" { - destdir = "/host/tmp" - } - - platformType := utils.Baremetal + nodeWriter := daemon.NewNodeStateStatusWriter(writerclient, + closeAllConns, + eventRecorder, + hostHelpers, + platformHelper) nodeInfo, err := kubeclient.CoreV1().Nodes().Get(context.Background(), startOpts.nodeName, v1.GetOptions{}) if err == nil { - for key, pType := range utils.PlatformMap { + for key, pType := range vars.PlatformsMap { if strings.Contains(strings.ToLower(nodeInfo.Spec.ProviderID), strings.ToLower(key)) { - platformType = pType + vars.PlatformType = pType } } } else { setupLog.Error(err, "failed to fetch node state, exiting", "node-name", startOpts.nodeName) return err } - setupLog.Info("Running on", "platform", platformType.String()) + setupLog.Info("Running on", "platform", vars.PlatformType.String()) var namespace = os.Getenv("NAMESPACE") if err := sriovnetworkv1.InitNicIDMapFromConfigMap(kubeclient, namespace); err != nil { @@ -195,27 +225,24 @@ func runStartCmd(cmd *cobra.Command, args []string) error { eventRecorder.SendEvent("ConfigDaemonStart", "Config Daemon starting") // block the deamon process until nodeWriter finish first its run - err = nodeWriter.RunOnce(destdir, platformType) + err = nodeWriter.RunOnce() if err != nil { setupLog.Error(err, "failed to run writer") return err } - go nodeWriter.Run(stopCh, refreshCh, syncCh, platformType) + go nodeWriter.Run(stopCh, refreshCh, syncCh) setupLog.V(0).Info("Starting SriovNetworkConfigDaemon") err = daemon.New( - startOpts.nodeName, snclient, kubeclient, - openshiftContext, + hostHelpers, + platformHelper, exitCh, stopCh, syncCh, refreshCh, - platformType, - startOpts.systemd, eventRecorder, - devMode, ).Run(stopCh, exitCh) if err != nil { setupLog.Error(err, "failed to run daemon") diff --git a/controllers/sriovnetworkpoolconfig_controller.go b/controllers/sriovnetworkpoolconfig_controller.go index de8fb32b2..fd4643476 100644 --- a/controllers/sriovnetworkpoolconfig_controller.go +++ b/controllers/sriovnetworkpoolconfig_controller.go @@ -17,15 +17,16 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) // SriovNetworkPoolConfigReconciler reconciles a SriovNetworkPoolConfig object type SriovNetworkPoolConfigReconciler struct { client.Client - Scheme *runtime.Scheme - OpenshiftContext *utils.OpenshiftContext + Scheme *runtime.Scheme + PlatformHelper platforms.Interface } //+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs,verbs=get;list;watch;create;update;patch;delete @@ -44,8 +45,8 @@ type SriovNetworkPoolConfigReconciler struct { func (r *SriovNetworkPoolConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues("sriovnetworkpoolconfig", req.NamespacedName) isHypershift := false - if r.OpenshiftContext.IsOpenshiftCluster() { - if r.OpenshiftContext.IsHypershift() { + if r.PlatformHelper.IsOpenshiftCluster() { + if r.PlatformHelper.IsHypershift() { isHypershift = true } logger = logger.WithValues("isHypershift", isHypershift) @@ -78,7 +79,7 @@ func (r *SriovNetworkPoolConfigReconciler) Reconcile(ctx context.Context, req ct return reconcile.Result{}, err } } - if utils.ClusterType == utils.ClusterTypeOpenshift { + if vars.ClusterType == constants.ClusterTypeOpenshift { if !isHypershift { if err = r.syncOvsHardwareOffloadMachineConfigs(ctx, instance, false); err != nil { return reconcile.Result{}, err @@ -92,7 +93,7 @@ func (r *SriovNetworkPoolConfigReconciler) Reconcile(ctx context.Context, req ct if sriovnetworkv1.StringInArray(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) { // our finalizer is present, so lets handle any external dependency logger.Info("delete SriovNetworkPoolConfig CR", "Namespace", instance.Namespace, "Name", instance.Name) - if utils.ClusterType == utils.ClusterTypeOpenshift && !isHypershift { + if vars.ClusterType == constants.ClusterTypeOpenshift && !isHypershift { if err = r.syncOvsHardwareOffloadMachineConfigs(ctx, instance, true); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 4c453b6de..1e4c7728b 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -39,17 +39,19 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" apply "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + consts "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" utils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) // SriovOperatorConfigReconciler reconciles a SriovOperatorConfig object type SriovOperatorConfigReconciler struct { client.Client - Scheme *runtime.Scheme - OpenshiftContext *utils.OpenshiftContext + Scheme *runtime.Scheme + PlatformHelper platforms.Interface } //+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs,verbs=get;list;watch;create;update;patch;delete @@ -70,13 +72,12 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. logger.Info("Reconciling SriovOperatorConfig") - enableAdmissionController := os.Getenv("ADMISSION_CONTROLLERS_ENABLED") == trueString - if !enableAdmissionController { + if !vars.EnableAdmissionController { logger.Info("SR-IOV Network Resource Injector and Operator Webhook are disabled.") } defaultConfig := &sriovnetworkv1.SriovOperatorConfig{} err := r.Get(ctx, types.NamespacedName{ - Name: constants.DefaultConfigName, Namespace: namespace}, defaultConfig) + Name: consts.DefaultConfigName, Namespace: namespace}, defaultConfig) if err != nil { if apierrors.IsNotFound(err) { singleNode, err := utils.IsSingleNodeCluster(r.Client) @@ -86,10 +87,10 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. // Default Config object not found, create it. defaultConfig.SetNamespace(namespace) - defaultConfig.SetName(constants.DefaultConfigName) + defaultConfig.SetName(consts.DefaultConfigName) defaultConfig.Spec = sriovnetworkv1.SriovOperatorConfigSpec{ - EnableInjector: func() *bool { b := enableAdmissionController; return &b }(), - EnableOperatorWebhook: func() *bool { b := enableAdmissionController; return &b }(), + EnableInjector: func() *bool { b := vars.EnableAdmissionController; return &b }(), + EnableOperatorWebhook: func() *bool { b := vars.EnableAdmissionController; return &b }(), ConfigDaemonNodeSelector: map[string]string{}, LogLevel: 2, DisableDrain: singleNode, @@ -99,7 +100,7 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. err = r.Create(ctx, defaultConfig) if err != nil { logger.Error(err, "Failed to create default Operator Config", "Namespace", - namespace, "Name", constants.DefaultConfigName) + namespace, "Name", consts.DefaultConfigName) return reconcile.Result{}, err } return reconcile.Result{}, nil @@ -129,9 +130,9 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. snolog.SetLogLevel(defaultConfig.Spec.LogLevel) // For Openshift we need to create the systemd files using a machine config - if utils.ClusterType == utils.ClusterTypeOpenshift { + if vars.ClusterType == consts.ClusterTypeOpenshift { // TODO: add support for hypershift as today there is no MCO on hypershift clusters - if r.OpenshiftContext.IsHypershift() { + if r.PlatformHelper.IsHypershift() { return ctrl.Result{}, fmt.Errorf("systemd mode is not supported on hypershift") } @@ -139,7 +140,7 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. return reconcile.Result{}, err } } - return reconcile.Result{RequeueAfter: constants.ResyncPeriod}, nil + return reconcile.Result{RequeueAfter: consts.ResyncPeriod}, nil } // SetupWithManager sets up the controller with the Manager. @@ -192,7 +193,7 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context, data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE") data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") - data.Data["ClusterType"] = utils.ClusterType + data.Data["ClusterType"] = vars.ClusterType data.Data["DevMode"] = os.Getenv("DEV_MODE") data.Data["ImagePullSecrets"] = GetImagePullSecrets() if dc.Spec.ConfigurationMode == sriovnetworkv1.SystemdConfigurationMode { @@ -208,7 +209,7 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context, logger.V(1).Info("New cni bin found", "CNIBinPath", envCniBinPath) data.Data["CNIBinPath"] = envCniBinPath } - objs, err := render.RenderDir(constants.ConfigDaemonPath, &data) + objs, err := render.RenderDir(consts.ConfigDaemonPath, &data) if err != nil { logger.Error(err, "Fail to render config daemon manifests") return err @@ -251,7 +252,7 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc data.Data["NetworkResourcesInjectorImage"] = os.Getenv("NETWORK_RESOURCES_INJECTOR_IMAGE") data.Data["SriovNetworkWebhookImage"] = os.Getenv("SRIOV_NETWORK_WEBHOOK_IMAGE") data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") - data.Data["ClusterType"] = utils.ClusterType + data.Data["ClusterType"] = vars.ClusterType data.Data["DevMode"] = os.Getenv("DEV_MODE") data.Data["ImagePullSecrets"] = GetImagePullSecrets() data.Data["CertManagerEnabled"] = strings.ToLower(os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED")) == trueString @@ -261,8 +262,8 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc data.Data["InjectorWebhookCA"] = os.Getenv("ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT") data.Data["ExternalControlPlane"] = false - if r.OpenshiftContext.IsOpenshiftCluster() { - external := r.OpenshiftContext.IsHypershift() + if r.PlatformHelper.IsOpenshiftCluster() { + external := r.PlatformHelper.IsHypershift() data.Data["ExternalControlPlane"] = external } @@ -273,7 +274,7 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc } // Delete injector webhook - if !*dc.Spec.EnableInjector && path == constants.InjectorWebHookPath { + if !*dc.Spec.EnableInjector && path == consts.InjectorWebHookPath { for _, obj := range objs { err = r.deleteWebhookObject(ctx, obj) if err != nil { @@ -286,7 +287,7 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc continue } // Delete operator webhook - if !*dc.Spec.EnableOperatorWebhook && path == constants.OperatorWebHookPath { + if !*dc.Spec.EnableOperatorWebhook && path == consts.OperatorWebHookPath { for _, obj := range objs { err = r.deleteWebhookObject(ctx, obj) if err != nil { @@ -347,7 +348,7 @@ func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context. if cr.Spec.ConfigurationMode != sriovnetworkv1.SystemdConfigurationMode { obj := &machinev1.MachineConfig{} - err := r.Get(context.TODO(), types.NamespacedName{Name: constants.SystemdServiceOcpMachineConfigName}, obj) + err := r.Get(context.TODO(), types.NamespacedName{Name: consts.SystemdServiceOcpMachineConfigName}, obj) if err != nil { if apierrors.IsNotFound(err) { return nil @@ -370,7 +371,7 @@ func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context. logger.Info("Start to sync config systemd machine config for openshift") data := render.MakeRenderData() data.Data["LogLevel"] = cr.Spec.LogLevel - objs, err := render.RenderDir(constants.SystemdServiceOcpPath, &data) + objs, err := render.RenderDir(consts.SystemdServiceOcpPath, &data) if err != nil { logger.Error(err, "Fail to render config daemon manifests") return err diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 4fe8a3eb8..0493ab7c6 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -23,13 +23,11 @@ import ( "testing" "time" - netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - openshiftconfigv1 "github.com/openshift/api/config/v1" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" - "go.uber.org/zap/zapcore" + "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" @@ -39,10 +37,15 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + openshiftconfigv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + + //+kubebuilder:scaffold:imports sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" - //+kubebuilder:scaffold:imports + mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -129,17 +132,24 @@ var _ = BeforeSuite(func(done Done) { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + t := GinkgoT() + mockCtrl := gomock.NewController(t) + platformHelper := mock_platforms.NewMockInterface(mockCtrl) + platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes() + platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes() + platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes() + err = (&SriovOperatorConfigReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - OpenshiftContext: &utils.OpenshiftContext{OpenshiftFlavor: utils.OpenshiftFlavorDefault}, + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + PlatformHelper: platformHelper, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) err = (&SriovNetworkPoolConfigReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - OpenshiftContext: &utils.OpenshiftContext{OpenshiftFlavor: utils.OpenshiftFlavorDefault}, + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + PlatformHelper: platformHelper, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/main.go b/main.go index 7d62af130..07f3f70be 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,9 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "k8s.io/apimachinery/pkg/api/errors" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -90,12 +93,6 @@ func main() { os.Exit(1) } - openshiftContext, err := utils.NewOpenshiftContext(restConfig, scheme) - if err != nil { - setupLog.Error(err, "couldn't create openshift context") - os.Exit(1) - } - le := leaderelection.GetLeaderElectionConfig(kubeClient, enableLeaderElection) namespace := os.Getenv("NAMESPACE") @@ -137,6 +134,16 @@ func main() { os.Exit(1) } + // Initial global info + vars.Config = restConfig + vars.Scheme = mgrGlobal.GetScheme() + + platformsHelper, err := platforms.NewDefaultPlatformHelper() + if err != nil { + setupLog.Error(err, "couldn't create openshift context") + os.Exit(1) + } + if err = (&controllers.SriovNetworkReconciler{ Client: mgrGlobal.GetClient(), Scheme: mgrGlobal.GetScheme(), @@ -159,17 +166,17 @@ func main() { os.Exit(1) } if err = (&controllers.SriovOperatorConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - OpenshiftContext: openshiftContext, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PlatformHelper: platformsHelper, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SriovOperatorConfig") os.Exit(1) } if err = (&controllers.SriovNetworkPoolConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - OpenshiftContext: openshiftContext, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PlatformHelper: platformsHelper, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SriovNetworkPoolConfig") os.Exit(1) diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index 1f37edfa4..d56d36e7f 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -1,8 +1,19 @@ package consts -import "time" +import ( + "fmt" + "time" +) + +type DrainState string + +// PlatformTypes +type PlatformTypes int const ( + Chroot = "/host" + Host = "/host" + ResyncPeriod = 5 * time.Minute DefaultConfigName = "default" ConfigDaemonPath = "./bindata/manifests/daemon" @@ -33,4 +44,72 @@ const ( DeviceTypeNetDevice = "netdevice" VdpaTypeVirtio = "virtio" VdpaTypeVhost = "vhost" + + ClusterTypeOpenshift = "openshift" + ClusterTypeKubernetes = "kubernetes" + + SriovConfBasePath = "/etc/sriov-operator" + PfAppliedConfig = SriovConfBasePath + "/pci" + SriovSwitchDevConfPath = SriovConfBasePath + "/sriov_config.json" + SriovHostSwitchDevConfPath = Host + SriovSwitchDevConfPath + + DrainAnnotationState = "sriovnetwork.openshift.io/state" + DrainAnnotationStateRequired = "sriovnetwork.openshift.io/state-required" + DrainAnnotationTime = "sriovnetwork.openshift.io/state-time" + + DrainIdle DrainState = "Idle" + DrainDisabled DrainState = "Drain_Disabled" + DrainRequired DrainState = "Drain_Required" + RebootRequired DrainState = "Reboot_Required" + DrainMcpPausing DrainState = "Draining_MCP_Pausing" + DrainMcpPaused DrainState = "Draining_MCP_Paused" + Draining DrainState = "Draining" + DrainingComplete DrainState = "Draining_Complete" + RebootComplete DrainState = "Reboot_Complete" + + SyncStatusSucceeded = "Succeeded" + SyncStatusFailed = "Failed" + SyncStatusInProgress = "InProgress" + + MCPPauseAnnotationState = "sriovnetwork.openshift.io/state" + MCPPauseAnnotationTime = "sriovnetwork.openshift.io/time" + + CheckpointFileName = "sno-initial-node-state.json" + Unknown = "Unknown" + + SysBusPciDevices = "/sys/bus/pci/devices" + SysBusPciDrivers = "/sys/bus/pci/drivers" + SysBusPciDriversProbe = "/sys/bus/pci/drivers_probe" + SysClassNet = "/sys/class/net" + ProcKernelCmdLine = "/proc/cmdline" + NetClass = 0x02 + NumVfsFile = "sriov_numvfs" + + UdevFolder = "/etc/udev" + UdevRulesFolder = UdevFolder + "/rules.d" + HostUdevRulesFolder = Host + UdevRulesFolder + UdevDisableNM = "/bindata/scripts/udev-find-sriov-pf.sh" + NMUdevRule = "SUBSYSTEM==\"net\", ACTION==\"add|change|move\", ATTRS{device}==\"%s\", IMPORT{program}=\"/etc/udev/disable-nm-sriov.sh $env{INTERFACE} %s\"" + + KernelArgPciRealloc = "pci=realloc" + KernelArgIntelIommu = "intel_iommu=on" + KernelArgIommuPt = "iommu=pt" ) + +const ( + // Baremetal platform + Baremetal PlatformTypes = iota + // VirtualOpenStack platform + VirtualOpenStack +) + +func (e PlatformTypes) String() string { + switch e { + case Baremetal: + return "Baremetal" + case VirtualOpenStack: + return "Virtual/Openstack" + default: + return fmt.Sprintf("%d", int(e)) + } +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 913cad805..290b4400e 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -39,12 +39,13 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" sninformer "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) const ( @@ -62,26 +63,17 @@ type Message struct { } type Daemon struct { - // name is the node name. - name string - - platform utils.PlatformType - - useSystemdService bool - - devMode bool - client snclientset.Interface // kubeClient allows interaction with Kubernetes, including the node we are running on. kubeClient kubernetes.Interface - openshiftContext *utils.OpenshiftContext - nodeState *sriovnetworkv1.SriovNetworkNodeState enabledPlugins map[string]plugin.VendorPlugin - serviceManager service.ServiceManager + HostHelpers helper.HostHelpersInterface + + platformHelpers platforms.Interface // channel used by callbacks to signal Run() of an error exitCh chan<- error @@ -109,22 +101,15 @@ type Daemon struct { mcpName string - storeManager utils.StoreManagerInterface - - hostManager host.HostManagerInterface - eventRecorder *EventRecorder } const ( - udevScriptsPath = "/bindata/scripts/load-udev.sh" - annoKey = "sriovnetwork.openshift.io/state" - annoIdle = "Idle" - annoDraining = "Draining" - annoMcpPaused = "Draining_MCP_Paused" - syncStatusSucceeded = "Succeeded" - syncStatusFailed = "Failed" - syncStatusInProgress = "InProgress" + udevScriptsPath = "/bindata/scripts/load-udev.sh" + annoKey = "sriovnetwork.openshift.io/state" + annoIdle = "Idle" + annoDraining = "Draining" + annoMcpPaused = "Draining_MCP_Paused" ) var namespace = os.Getenv("NAMESPACE") @@ -141,33 +126,26 @@ func (w writer) Write(p []byte) (n int, err error) { } func New( - nodeName string, client snclientset.Interface, kubeClient kubernetes.Interface, - openshiftContext *utils.OpenshiftContext, + hostHelpers helper.HostHelpersInterface, + platformHelper platforms.Interface, exitCh chan<- error, stopCh <-chan struct{}, syncCh <-chan struct{}, refreshCh chan<- Message, - platformType utils.PlatformType, - useSystemdService bool, er *EventRecorder, - devMode bool, ) *Daemon { return &Daemon{ - name: nodeName, - platform: platformType, - useSystemdService: useSystemdService, - devMode: devMode, - client: client, - kubeClient: kubeClient, - openshiftContext: openshiftContext, - serviceManager: service.NewServiceManager("/host"), - exitCh: exitCh, - stopCh: stopCh, - syncCh: syncCh, - refreshCh: refreshCh, - nodeState: &sriovnetworkv1.SriovNetworkNodeState{}, + client: client, + kubeClient: kubeClient, + HostHelpers: hostHelpers, + platformHelpers: platformHelper, + exitCh: exitCh, + stopCh: stopCh, + syncCh: syncCh, + refreshCh: refreshCh, + nodeState: &sriovnetworkv1.SriovNetworkNodeState{}, drainer: &drain.Helper{ Client: kubeClient, Force: true, @@ -195,38 +173,30 @@ func New( // Run the config daemon func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { - log.Log.V(0).Info("Run()", "node", dn.name) + log.Log.V(0).Info("Run()", "node", vars.NodeName) - if utils.ClusterType == utils.ClusterTypeOpenshift { - log.Log.V(0).Info("Run(): start daemon.", "openshiftFlavor", dn.openshiftContext.OpenshiftFlavor) + if vars.ClusterType == consts.ClusterTypeOpenshift { + log.Log.V(0).Info("Run(): start daemon.", "openshiftFlavor", dn.platformHelpers.GetFlavor()) } else { log.Log.V(0).Info("Run(): start daemon.") } - if dn.useSystemdService { - log.Log.V(0).Info("Run(): daemon running in systemd mode") - } - // Only watch own SriovNetworkNodeState CR - defer utilruntime.HandleCrash() - defer dn.workqueue.ShutDown() - - hostManager := host.NewHostManager(dn.useSystemdService) - dn.hostManager = hostManager - if !dn.useSystemdService { - dn.hostManager.TryEnableRdma() - dn.hostManager.TryEnableTun() - dn.hostManager.TryEnableVhostNet() - err := systemd.CleanSriovFilesFromHost(utils.ClusterType == utils.ClusterTypeOpenshift) + if !vars.UsingSystemdMode { + log.Log.V(0).Info("Run(): daemon running in daemon mode") + dn.HostHelpers.TryEnableRdma() + dn.HostHelpers.TryEnableTun() + dn.HostHelpers.TryEnableVhostNet() + err := systemd.CleanSriovFilesFromHost(vars.ClusterType == consts.ClusterTypeOpenshift) if err != nil { log.Log.Error(err, "failed to remove all the systemd sriov files") } + } else { + log.Log.V(0).Info("Run(): daemon running in systemd mode") } - storeManager, err := utils.NewStoreManager(false) - if err != nil { - return err - } - dn.storeManager = storeManager + // Only watch own SriovNetworkNodeState CR + defer utilruntime.HandleCrash() + defer dn.workqueue.ShutDown() if err := dn.prepareNMUdevRule(); err != nil { log.Log.Error(err, "failed to prepare udev files to disable network manager on requested VFs") @@ -241,7 +211,7 @@ func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { time.Second*15, namespace, func(lo *metav1.ListOptions) { - lo.FieldSelector = "metadata.name=" + dn.name + lo.FieldSelector = "metadata.name=" + vars.NodeName lo.TimeoutSeconds = &timeout }, ) @@ -300,7 +270,7 @@ func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { log.Log.Error(err, "got an error") if more { dn.refreshCh <- Message{ - syncStatus: syncStatusFailed, + syncStatus: consts.SyncStatusFailed, lastSyncError: err.Error(), } } @@ -358,7 +328,7 @@ func (dn *Daemon) processNextWorkItem() bool { if err != nil { // Ereport error message, and put the item back to work queue for retry. dn.refreshCh <- Message{ - syncStatus: syncStatusFailed, + syncStatus: consts.SyncStatusFailed, lastSyncError: err.Error(), } <-dn.syncCh @@ -384,9 +354,9 @@ func (dn *Daemon) nodeAddHandler(obj interface{}) { } func (dn *Daemon) nodeUpdateHandler(old, new interface{}) { - node, err := dn.nodeLister.Get(dn.name) + node, err := dn.nodeLister.Get(vars.NodeName) if errors.IsNotFound(err) { - log.Log.V(2).Info("nodeUpdateHandler(): node has been deleted", "name", dn.name) + log.Log.V(2).Info("nodeUpdateHandler(): node has been deleted", "name", vars.NodeName) return } dn.node = node.DeepCopy() @@ -399,7 +369,7 @@ func (dn *Daemon) nodeUpdateHandler(old, new interface{}) { // Checking if other nodes are draining for _, otherNode := range nodes { - if otherNode.GetName() == dn.name { + if otherNode.GetName() == vars.NodeName { continue } @@ -438,24 +408,24 @@ func (dn *Daemon) nodeStateSyncHandler() error { var err error // Get the latest NodeState var latestState *sriovnetworkv1.SriovNetworkNodeState - var sriovResult = &systemd.SriovResult{SyncStatus: syncStatusSucceeded, LastSyncError: ""} - latestState, err = dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), dn.name, metav1.GetOptions{}) + var sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusSucceeded, LastSyncError: ""} + latestState, err = dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", dn.name) + log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", vars.NodeName) return err } latest := latestState.GetGeneration() log.Log.V(0).Info("nodeStateSyncHandler(): new generation", "generation", latest) - if utils.ClusterType == utils.ClusterTypeOpenshift && !dn.openshiftContext.IsHypershift() { + if vars.ClusterType == consts.ClusterTypeOpenshift && !dn.platformHelpers.IsHypershift() { if err = dn.getNodeMachinePool(); err != nil { return err } } if dn.nodeState.GetGeneration() == latest { - if dn.useSystemdService { - serviceEnabled, err := dn.serviceManager.IsServiceEnabled(systemd.SriovServicePath) + if vars.UsingSystemdMode { + serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) if err != nil { log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") return err @@ -465,7 +435,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply // the system service and reboot the node the config-daemon doesn't need to do anything. if !serviceEnabled { - sriovResult = &systemd.SriovResult{SyncStatus: syncStatusFailed, + sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, LastSyncError: "sriov-config systemd service is not available on node"} } else { sriovResult, err = systemd.ReadSriovResult() @@ -474,12 +444,12 @@ func (dn *Daemon) nodeStateSyncHandler() error { return err } } - if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == syncStatusFailed { + if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) // add the error but don't requeue dn.refreshCh <- Message{ - syncStatus: syncStatusFailed, + syncStatus: consts.SyncStatusFailed, lastSyncError: sriovResult.LastSyncError, } <-dn.syncCh @@ -488,9 +458,9 @@ func (dn *Daemon) nodeStateSyncHandler() error { } log.Log.V(0).Info("nodeStateSyncHandler(): Interface not changed") if latestState.Status.LastSyncError != "" || - latestState.Status.SyncStatus != syncStatusSucceeded { + latestState.Status.SyncStatus != consts.SyncStatusSucceeded { dn.refreshCh <- Message{ - syncStatus: syncStatusSucceeded, + syncStatus: consts.SyncStatusSucceeded, lastSyncError: "", } // wait for writer to refresh the status @@ -501,7 +471,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } if latestState.GetGeneration() == 1 && len(latestState.Spec.Interfaces) == 0 { - err = dn.storeManager.ClearPCIAddressFolder() + err = dn.HostHelpers.ClearPCIAddressFolder() if err != nil { log.Log.Error(err, "failed to clear the PCI address configuration") return err @@ -522,7 +492,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } dn.refreshCh <- Message{ - syncStatus: syncStatusInProgress, + syncStatus: consts.SyncStatusInProgress, lastSyncError: "", } // wait for writer to refresh status then pull again the latest node state @@ -531,16 +501,16 @@ func (dn *Daemon) nodeStateSyncHandler() error { // we need to load the latest status to our object // if we don't do it we can have a race here where the user remove the virtual functions but the operator didn't // trigger the refresh - updatedState, err := dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), dn.name, metav1.GetOptions{}) + updatedState, err := dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", dn.name) + log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", vars.NodeName) return err } latestState.Status = updatedState.Status // load plugins if it has not loaded if len(dn.enabledPlugins) == 0 { - dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, latestState, dn.hostManager, dn.storeManager) + dn.enabledPlugins, err = enablePlugins(latestState, dn.HostHelpers) if err != nil { log.Log.Error(err, "nodeStateSyncHandler(): failed to enable vendor plugins") return err @@ -571,8 +541,9 @@ func (dn *Daemon) nodeStateSyncHandler() error { // When running using systemd check if the applied configuration is the latest one // or there is a new config we need to apply // When using systemd configuration we write the file - if dn.useSystemdService { - systemdConfModified, err := systemd.WriteConfFile(latestState, dn.devMode, dn.platform) + if vars.UsingSystemdMode { + log.Log.V(0).Info("nodeStateSyncHandler(): writing systemd config file to host") + systemdConfModified, err := systemd.WriteConfFile(latestState) if err != nil { log.Log.Error(err, "nodeStateSyncHandler(): failed to write configuration file for systemd mode") return err @@ -610,7 +581,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } } - if dn.openshiftContext.IsOpenshiftCluster() && !dn.openshiftContext.IsHypershift() { + if dn.platformHelpers.IsOpenshiftCluster() && !dn.platformHelpers.IsHypershift() { if err = dn.getNodeMachinePool(); err != nil { return err } @@ -628,7 +599,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } - if dn.openshiftContext.IsOpenshiftCluster() && !dn.openshiftContext.IsHypershift() { + if dn.platformHelpers.IsOpenshiftCluster() && !dn.platformHelpers.IsHypershift() { log.Log.Info("nodeStateSyncHandler(): pause MCP") if err := dn.pauseMCP(); err != nil { return err @@ -645,7 +616,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } - if !reqReboot && !dn.useSystemdService { + if !reqReboot && !vars.UsingSystemdMode { // For BareMetal machines apply the generic plugin selectedPlugin, ok := dn.enabledPlugins[GenericPluginName] if ok { @@ -672,7 +643,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { if reqReboot { log.Log.Info("nodeStateSyncHandler(): reboot node") dn.eventRecorder.SendEvent("RebootNode", "Reboot node has been initiated") - rebootNode() + dn.rebootNode() return nil } @@ -689,7 +660,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } else { if !dn.nodeHasAnnotation(annoKey, annoIdle) { - if err := dn.annotateNode(dn.name, annoIdle); err != nil { + if err := dn.annotateNode(vars.NodeName, annoIdle); err != nil { log.Log.Error(err, "nodeStateSyncHandler(): failed to annotate node") return err } @@ -697,14 +668,14 @@ func (dn *Daemon) nodeStateSyncHandler() error { } log.Log.Info("nodeStateSyncHandler(): sync succeeded") dn.nodeState = latestState.DeepCopy() - if dn.useSystemdService { + if vars.UsingSystemdMode { dn.refreshCh <- Message{ syncStatus: sriovResult.SyncStatus, lastSyncError: sriovResult.LastSyncError, } } else { dn.refreshCh <- Message{ - syncStatus: syncStatusSucceeded, + syncStatus: consts.SyncStatusSucceeded, lastSyncError: "", } } @@ -739,16 +710,16 @@ func (dn *Daemon) completeDrain() error { } } - if dn.openshiftContext.IsOpenshiftCluster() && !dn.openshiftContext.IsHypershift() { + if dn.platformHelpers.IsOpenshiftCluster() && !dn.platformHelpers.IsHypershift() { log.Log.Info("completeDrain(): resume MCP", "mcp-name", dn.mcpName) pausePatch := []byte("{\"spec\":{\"paused\":false}}") - if _, err := dn.openshiftContext.McClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}); err != nil { + if _, err := dn.platformHelpers.GetMcClient().MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}); err != nil { log.Log.Error(err, "completeDrain(): failed to resume MCP", "mcp-name", dn.mcpName) return err } } - if err := dn.annotateNode(dn.name, annoIdle); err != nil { + if err := dn.annotateNode(vars.NodeName, annoIdle); err != nil { log.Log.Error(err, "completeDrain(): failed to annotate node") return err } @@ -763,7 +734,7 @@ func (dn *Daemon) restartDevicePluginPod() error { var podToDelete string pods, err := dn.kubeClient.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ LabelSelector: "app=sriov-device-plugin", - FieldSelector: "spec.nodeName=" + dn.name, + FieldSelector: "spec.nodeName=" + vars.NodeName, }) if err != nil { if errors.IsNotFound(err) { @@ -812,9 +783,9 @@ func (dn *Daemon) restartDevicePluginPod() error { return nil } -func rebootNode() { +func (dn *Daemon) rebootNode() { log.Log.Info("rebootNode(): trigger node reboot") - exit, err := utils.Chroot("/host") + exit, err := dn.HostHelpers.Chroot(consts.Host) if err != nil { log.Log.Error(err, "rebootNode(): chroot command failed") } @@ -837,7 +808,7 @@ func rebootNode() { func (dn *Daemon) annotateNode(node, value string) error { log.Log.Info("annotateNode(): Annotate node", "name", node, "value", value) - oldNode, err := dn.kubeClient.CoreV1().Nodes().Get(context.Background(), dn.name, metav1.GetOptions{}) + oldNode, err := dn.kubeClient.CoreV1().Nodes().Get(context.Background(), vars.NodeName, metav1.GetOptions{}) if err != nil { log.Log.Error(err, "annotateNode(): Failed to get node, retrying", "name", node) return err @@ -863,7 +834,7 @@ func (dn *Daemon) annotateNode(node, value string) error { return err } _, err = dn.kubeClient.CoreV1().Nodes().Patch(context.Background(), - dn.name, + vars.NodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) @@ -881,7 +852,7 @@ func (dn *Daemon) getNodeMachinePool() error { log.Log.Error(nil, "getNodeMachinePool(): Failed to find the the desiredConfig Annotation") return fmt.Errorf("getNodeMachinePool(): Failed to find the the desiredConfig Annotation") } - mc, err := dn.openshiftContext.McClient.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), desiredConfig, metav1.GetOptions{}) + mc, err := dn.platformHelpers.GetMcClient().MachineconfigurationV1().MachineConfigs().Get(context.TODO(), desiredConfig, metav1.GetOptions{}) if err != nil { log.Log.Error(err, "getNodeMachinePool(): Failed to get the desired Machine Config") return err @@ -907,7 +878,7 @@ func (dn *Daemon) getDrainLock(ctx context.Context, done chan bool) { }, Client: dn.kubeClient.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{ - Identity: dn.name, + Identity: vars.NodeName, }, } @@ -930,7 +901,7 @@ func (dn *Daemon) getDrainLock(ctx context.Context, done chan bool) { } if dn.drainable { log.Log.V(2).Info("getDrainLock(): no other node is draining") - err = dn.annotateNode(dn.name, annoDraining) + err = dn.annotateNode(vars.NodeName, annoDraining) if err != nil { log.Log.Error(err, "getDrainLock(): failed to annotate node") continue @@ -952,7 +923,7 @@ func (dn *Daemon) pauseMCP() error { log.Log.Info("pauseMCP(): pausing MCP") var err error - mcpInformerFactory := mcfginformers.NewSharedInformerFactory(dn.openshiftContext.McClient, + mcpInformerFactory := mcfginformers.NewSharedInformerFactory(dn.platformHelpers.GetMcClient(), time.Second*30, ) mcpInformer := mcpInformerFactory.Machineconfiguration().V1().MachineConfigPools().Informer() @@ -967,7 +938,7 @@ func (dn *Daemon) pauseMCP() error { return } // Always get the latest object - newMcp, err := dn.openshiftContext.McClient.MachineconfigurationV1().MachineConfigPools().Get(ctx, dn.mcpName, metav1.GetOptions{}) + newMcp, err := dn.platformHelpers.GetMcClient().MachineconfigurationV1().MachineConfigPools().Get(ctx, dn.mcpName, metav1.GetOptions{}) if err != nil { log.Log.V(2).Error(err, "pauseMCP(): Failed to get MCP", "mcp-name", dn.mcpName) return @@ -987,12 +958,12 @@ func (dn *Daemon) pauseMCP() error { } log.Log.Info("pauseMCP(): pause MCP", "mcp-name", dn.mcpName) pausePatch := []byte("{\"spec\":{\"paused\":true}}") - _, err = dn.openshiftContext.McClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) + _, err = dn.platformHelpers.GetMcClient().MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) if err != nil { log.Log.V(2).Error(err, "pauseMCP(): failed to pause MCP", "mcp-name", dn.mcpName) return } - err = dn.annotateNode(dn.name, annoMcpPaused) + err = dn.annotateNode(vars.NodeName, annoMcpPaused) if err != nil { log.Log.V(2).Error(err, "pauseMCP(): Failed to annotate node") return @@ -1003,12 +974,12 @@ func (dn *Daemon) pauseMCP() error { if paused { log.Log.Info("pauseMCP(): MCP is processing, resume MCP", "mcp-name", dn.mcpName) pausePatch := []byte("{\"spec\":{\"paused\":false}}") - _, err = dn.openshiftContext.McClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) + _, err = dn.platformHelpers.GetMcClient().MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) if err != nil { log.Log.V(2).Error(err, "pauseMCP(): fail to resume MCP", "mcp-name", dn.mcpName) return } - err = dn.annotateNode(dn.name, annoDraining) + err = dn.annotateNode(vars.NodeName, annoDraining) if err != nil { log.Log.V(2).Error(err, "pauseMCP(): Failed to annotate node") return @@ -1057,7 +1028,7 @@ func (dn *Daemon) drainNode() error { log.Log.Error(err, "cordon failed, retrying") return false, nil } - err = drain.RunNodeDrain(dn.drainer, dn.name) + err = drain.RunNodeDrain(dn.drainer, vars.NodeName) if err == nil { return true, nil } @@ -1077,28 +1048,29 @@ func (dn *Daemon) drainNode() error { return nil } +// TODO: move this to host interface func (dn *Daemon) tryCreateSwitchdevUdevRule() error { log.Log.V(2).Info("tryCreateSwitchdevUdevRule()") nodeState, nodeStateErr := dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get( context.Background(), - dn.name, + vars.NodeName, metav1.GetOptions{}, ) if nodeStateErr != nil { - log.Log.Error(nodeStateErr, "could not fetch node state, skip updating switchdev udev rules", "name", dn.name) + log.Log.Error(nodeStateErr, "could not fetch node state, skip updating switchdev udev rules", "name", vars.NodeName) return nil } var newContent string - filePath := path.Join(utils.FilesystemRoot, "/host/etc/udev/rules.d/20-switchdev.rules") + filePath := path.Join(vars.FilesystemRoot, "/host/etc/udev/rules.d/20-switchdev.rules") for _, ifaceStatus := range nodeState.Status.Interfaces { if ifaceStatus.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - switchID, err := utils.GetPhysSwitchID(ifaceStatus.Name) + switchID, err := dn.HostHelpers.GetPhysSwitchID(ifaceStatus.Name) if err != nil { return err } - portName, err := utils.GetPhysPortName(ifaceStatus.Name) + portName, err := dn.HostHelpers.GetPhysPortName(ifaceStatus.Name) if err != nil { return err } @@ -1126,7 +1098,7 @@ func (dn *Daemon) tryCreateSwitchdevUdevRule() error { } var stdout, stderr bytes.Buffer - cmd := exec.Command("/bin/bash", path.Join(utils.FilesystemRoot, udevScriptsPath)) + cmd := exec.Command("/bin/bash", path.Join(vars.FilesystemRoot, udevScriptsPath)) cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { @@ -1158,5 +1130,5 @@ func (dn *Daemon) prepareNMUdevRule() error { supportedVfIds = append(supportedVfIds, vfID) } - return utils.PrepareNMUdevRule(supportedVfIds) + return dn.HostHelpers.PrepareNMUdevRule(supportedVfIds) } diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go index 5300a1a65..b804888fa 100644 --- a/pkg/daemon/daemon_test.go +++ b/pkg/daemon/daemon_test.go @@ -5,6 +5,7 @@ import ( "flag" "testing" + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -12,14 +13,18 @@ import ( fakek8s "k8s.io/client-go/kubernetes/fake" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" fakesnclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/fake" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/fake" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) var FakeSupportedNicIDs corev1.ConfigMap = corev1.ConfigMap{ @@ -95,29 +100,44 @@ var _ = Describe("Config Daemon", func() { } var err error - utils.FilesystemRoot, cleanFakeFs, err = fakeFs.Use() + vars.FilesystemRoot, cleanFakeFs, err = fakeFs.Use() Expect(err).ToNot(HaveOccurred()) + vars.UsingSystemdMode = false + vars.NodeName = "test-node" + vars.PlatformType = consts.Baremetal + kubeClient := fakek8s.NewSimpleClientset(&FakeSupportedNicIDs, &SriovDevicePluginPod) client := fakesnclientset.NewSimpleClientset() err = sriovnetworkv1.InitNicIDMapFromConfigMap(kubeClient, namespace) Expect(err).ToNot(HaveOccurred()) - er := NewEventRecorder(client, "test-node", kubeClient) + er := NewEventRecorder(client, kubeClient) + + t := GinkgoT() + mockCtrl := gomock.NewController(t) + platformHelper := mock_platforms.NewMockInterface(mockCtrl) + platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes() + platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes() + platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes() - sut = New("test-node", + vendorHelper := mock_helper.NewMockHostHelpersInterface(mockCtrl) + vendorHelper.EXPECT().TryEnableRdma().Return(true, nil).AnyTimes() + vendorHelper.EXPECT().TryEnableVhostNet().AnyTimes() + vendorHelper.EXPECT().TryEnableTun().AnyTimes() + vendorHelper.EXPECT().PrepareNMUdevRule([]string{"0x1014", "0x154c"}).Return(nil).AnyTimes() + + sut = New( client, kubeClient, - &utils.OpenshiftContext{IsOpenShiftCluster: false, OpenshiftFlavor: ""}, + vendorHelper, + platformHelper, exitCh, stopCh, syncCh, refreshCh, - utils.Baremetal, - false, er, - false, ) sut.enabledPlugins = map[string]plugin.VendorPlugin{generic.PluginName: &fake.FakePlugin{}} @@ -236,66 +256,6 @@ var _ = Describe("Config Daemon", func() { Expect(sut.nodeState.GetGeneration()).To(BeNumerically("==", 777)) }) }) - - Context("isNodeDraining", func() { - - It("for a non-Openshift cluster", func() { - sut.openshiftContext = &utils.OpenshiftContext{IsOpenShiftCluster: false} - - sut.node = &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Annotations: map[string]string{}}} - - Expect(sut.isNodeDraining()).To(BeFalse()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining" - Expect(sut.isNodeDraining()).To(BeTrue()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining_MCP_Paused" - Expect(sut.isNodeDraining()).To(BeTrue()) - }) - - It("for an Openshift cluster", func() { - sut.openshiftContext = &utils.OpenshiftContext{ - IsOpenShiftCluster: true, - OpenshiftFlavor: utils.OpenshiftFlavorDefault, - } - - sut.node = &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Annotations: map[string]string{}}} - - Expect(sut.isNodeDraining()).To(BeFalse()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining" - Expect(sut.isNodeDraining()).To(BeTrue()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining_MCP_Paused" - Expect(sut.isNodeDraining()).To(BeTrue()) - }) - - It("for an Openshift Hypershift cluster", func() { - sut.openshiftContext = &utils.OpenshiftContext{ - IsOpenShiftCluster: true, - OpenshiftFlavor: utils.OpenshiftFlavorHypershift, - } - - sut.node = &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Annotations: map[string]string{}}} - - Expect(sut.isNodeDraining()).To(BeFalse()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining" - Expect(sut.isNodeDraining()).To(BeTrue()) - - sut.node.Annotations["sriovnetwork.openshift.io/state"] = "Draining_MCP_Paused" - Expect(sut.isNodeDraining()).To(BeTrue()) - }) - }) }) func createSriovNetworkNodeState(c snclientset.Interface, nodeState *sriovnetworkv1.SriovNetworkNodeState) error { diff --git a/pkg/daemon/event_recorder.go b/pkg/daemon/event_recorder.go index 2860cf84a..ed9b34ee7 100644 --- a/pkg/daemon/event_recorder.go +++ b/pkg/daemon/event_recorder.go @@ -12,24 +12,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) type EventRecorder struct { client snclientset.Interface - node string eventRecorder record.EventRecorder eventBroadcaster record.EventBroadcaster } // NewEventRecorder Create a new EventRecorder -func NewEventRecorder(c snclientset.Interface, n string, kubeclient kubernetes.Interface) *EventRecorder { +func NewEventRecorder(c snclientset.Interface, kubeclient kubernetes.Interface) *EventRecorder { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartStructuredLogging(4) eventBroadcaster.StartRecordingToSink(&typedv1core.EventSinkImpl{Interface: kubeclient.CoreV1().Events("")}) eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "config-daemon"}) return &EventRecorder{ client: c, - node: n, eventRecorder: eventRecorder, eventBroadcaster: eventBroadcaster, } @@ -37,9 +36,9 @@ func NewEventRecorder(c snclientset.Interface, n string, kubeclient kubernetes.I // SendEvent Send an Event on the NodeState object func (e *EventRecorder) SendEvent(eventType string, msg string) { - nodeState, err := e.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), e.node, metav1.GetOptions{}) + nodeState, err := e.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) if err != nil { - log.Log.V(2).Error(err, "SendEvent(): Failed to fetch node state, skip SendEvent", "name", e.node) + log.Log.V(2).Error(err, "SendEvent(): Failed to fetch node state, skip SendEvent", "name", vars.NodeName) return } e.eventRecorder.Event(nodeState, corev1.EventTypeNormal, eventType, msg) diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index 09c69271c..38e1d9d73 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -6,17 +6,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" genericplugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" intelplugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/intel" k8splugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/k8s" mellanoxplugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/mellanox" virtualplugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/virtual" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -var VendorPluginMap = map[string]func() (plugin.VendorPlugin, error){ +var VendorPluginMap = map[string]func(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error){ "8086": intelplugin.NewIntelPlugin, "15b3": mellanoxplugin.NewMellanoxPlugin, } @@ -29,33 +30,33 @@ var ( K8sPlugin = k8splugin.NewK8sPlugin ) -func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *sriovnetworkv1.SriovNetworkNodeState, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface) (map[string]plugin.VendorPlugin, error) { +func enablePlugins(ns *sriovnetworkv1.SriovNetworkNodeState, helpers helper.HostHelpersInterface) (map[string]plugin.VendorPlugin, error) { log.Log.Info("enableVendorPlugins(): enabling plugins") enabledPlugins := map[string]plugin.VendorPlugin{} - if platform == utils.VirtualOpenStack { - virtualPlugin, err := VirtualPlugin(false) + if vars.PlatformType == consts.VirtualOpenStack { + virtualPlugin, err := VirtualPlugin(helpers) if err != nil { log.Log.Error(err, "enableVendorPlugins(): failed to load the virtual plugin") return nil, err } enabledPlugins[virtualPlugin.Name()] = virtualPlugin } else { - enabledVendorPlugins, err := registerVendorPlugins(ns) + enabledVendorPlugins, err := registerVendorPlugins(ns, helpers) if err != nil { return nil, err } enabledPlugins = enabledVendorPlugins - if utils.ClusterType != utils.ClusterTypeOpenshift { - k8sPlugin, err := K8sPlugin(useSystemdService) + if vars.ClusterType != consts.ClusterTypeOpenshift { + k8sPlugin, err := K8sPlugin(helpers) if err != nil { log.Log.Error(err, "enableVendorPlugins(): failed to load the k8s plugin") return nil, err } enabledPlugins[k8sPlugin.Name()] = k8sPlugin } - genericPlugin, err := GenericPlugin(false, hostManager, storeManager) + genericPlugin, err := GenericPlugin(helpers) if err != nil { log.Log.Error(err, "enableVendorPlugins(): failed to load the generic plugin") return nil, err @@ -71,12 +72,12 @@ func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *srio return enabledPlugins, nil } -func registerVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]plugin.VendorPlugin, error) { +func registerVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState, helpers helper.HostHelpersInterface) (map[string]plugin.VendorPlugin, error) { vendorPlugins := map[string]plugin.VendorPlugin{} for _, iface := range ns.Status.Interfaces { if val, ok := VendorPluginMap[iface.Vendor]; ok { - plug, err := val() + plug, err := val(helpers) if err != nil { log.Log.Error(err, "registerVendorPlugins(): failed to load plugin", "plugin-name", plug.Name()) return vendorPlugins, fmt.Errorf("registerVendorPlugins(): failed to load the %s plugin error: %v", plug.Name(), err) diff --git a/pkg/daemon/writer.go b/pkg/daemon/writer.go index a9e65a417..1a83c77d8 100644 --- a/pkg/daemon/writer.go +++ b/pkg/daemon/writer.go @@ -16,7 +16,10 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) const ( @@ -25,62 +28,51 @@ const ( ) type NodeStateStatusWriter struct { - client snclientset.Interface - node string - status sriovnetworkv1.SriovNetworkNodeStateStatus - OnHeartbeatFailure func() - openStackDevicesInfo utils.OSPDevicesInfo - withUnsupportedDevices bool - storeManager utils.StoreManagerInterface - eventRecorder *EventRecorder + client snclientset.Interface + status sriovnetworkv1.SriovNetworkNodeStateStatus + OnHeartbeatFailure func() + platformHelper platforms.Interface + hostHelper helper.HostHelpersInterface + eventRecorder *EventRecorder } // NewNodeStateStatusWriter Create a new NodeStateStatusWriter -func NewNodeStateStatusWriter(c snclientset.Interface, n string, f func(), er *EventRecorder, devMode bool) *NodeStateStatusWriter { +func NewNodeStateStatusWriter(c snclientset.Interface, + f func(), er *EventRecorder, + hostHelper helper.HostHelpersInterface, + platformHelper platforms.Interface) *NodeStateStatusWriter { return &NodeStateStatusWriter{ - client: c, - node: n, - OnHeartbeatFailure: f, - eventRecorder: er, - withUnsupportedDevices: devMode, + client: c, + OnHeartbeatFailure: f, + eventRecorder: er, + hostHelper: hostHelper, + platformHelper: platformHelper, } } // RunOnce initial the interface status for both baremetal and virtual environments -func (w *NodeStateStatusWriter) RunOnce(destDir string, platformType utils.PlatformType) error { +func (w *NodeStateStatusWriter) RunOnce() error { log.Log.V(0).Info("RunOnce()") msg := Message{} - storeManager, err := utils.NewStoreManager(false) - if err != nil { - log.Log.Error(err, "failed to create store manager") - return err - } - w.storeManager = storeManager - - if platformType == utils.VirtualOpenStack { - ns, err := w.getCheckPointNodeState(destDir) + if vars.PlatformType == consts.VirtualOpenStack { + ns, err := w.getCheckPointNodeState() if err != nil { return err } if ns == nil { - metaData, networkData, err := utils.GetOpenstackData(true) - if err != nil { - log.Log.Error(err, "RunOnce(): failed to read OpenStack data") - } - - w.openStackDevicesInfo, err = utils.CreateOpenstackDevicesInfo(metaData, networkData) + err = w.platformHelper.CreateOpenstackDevicesInfo() if err != nil { return err } } else { - w.openStackDevicesInfo = utils.CreateOpenstackDevicesInfoFromNodeStatus(ns) + w.platformHelper.CreateOpenstackDevicesInfoFromNodeStatus(ns) } } log.Log.V(0).Info("RunOnce(): first poll for nic status") - if err := w.pollNicStatus(platformType); err != nil { + if err := w.pollNicStatus(); err != nil { log.Log.Error(err, "RunOnce(): first poll failed") } @@ -88,12 +80,12 @@ func (w *NodeStateStatusWriter) RunOnce(destDir string, platformType utils.Platf if err != nil { log.Log.Error(err, "RunOnce(): first writing to node status failed") } - return w.writeCheckpointFile(ns, destDir) + return w.writeCheckpointFile(ns) } // Run reads from the writer channel and sets the interface status. It will // return if the stop channel is closed. Intended to be run via a goroutine. -func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message, syncCh chan<- struct{}, platformType utils.PlatformType) error { +func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message, syncCh chan<- struct{}) error { log.Log.V(0).Info("Run(): start writer") msg := Message{} @@ -104,7 +96,7 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message return nil case msg = <-refresh: log.Log.V(0).Info("Run(): refresh trigger") - if err := w.pollNicStatus(platformType); err != nil { + if err := w.pollNicStatus(); err != nil { continue } _, err := w.setNodeStateStatus(msg) @@ -114,7 +106,7 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message syncCh <- struct{}{} case <-time.After(30 * time.Second): log.Log.V(2).Info("Run(): period refresh") - if err := w.pollNicStatus(platformType); err != nil { + if err := w.pollNicStatus(); err != nil { continue } w.setNodeStateStatus(msg) @@ -122,15 +114,15 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message } } -func (w *NodeStateStatusWriter) pollNicStatus(platformType utils.PlatformType) error { +func (w *NodeStateStatusWriter) pollNicStatus() error { log.Log.V(2).Info("pollNicStatus()") var iface []sriovnetworkv1.InterfaceExt var err error - if platformType == utils.VirtualOpenStack { - iface, err = utils.DiscoverSriovDevicesVirtual(w.openStackDevicesInfo) + if vars.PlatformType == consts.VirtualOpenStack { + iface, err = w.platformHelper.DiscoverSriovDevicesVirtual() } else { - iface, err = utils.DiscoverSriovDevices(w.withUnsupportedDevices, w.storeManager) + iface, err = w.hostHelper.DiscoverSriovDevices(w.hostHelper) } if err != nil { return err @@ -177,7 +169,7 @@ func (w *NodeStateStatusWriter) updateNodeStateStatusRetry(f func(*sriovnetworkv func (w *NodeStateStatusWriter) setNodeStateStatus(msg Message) (*sriovnetworkv1.SriovNetworkNodeState, error) { nodeState, err := w.updateNodeStateStatusRetry(func(nodeState *sriovnetworkv1.SriovNetworkNodeState) { nodeState.Status.Interfaces = w.status.Interfaces - if msg.lastSyncError != "" || msg.syncStatus == syncStatusSucceeded { + if msg.lastSyncError != "" || msg.syncStatus == consts.SyncStatusSucceeded { // clear lastSyncError when sync Succeeded nodeState.Status.LastSyncError = msg.lastSyncError } @@ -215,33 +207,33 @@ func (w *NodeStateStatusWriter) getNodeState() (*sriovnetworkv1.SriovNetworkNode var lastErr error var n *sriovnetworkv1.SriovNetworkNodeState err := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { - n, lastErr = w.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), w.node, metav1.GetOptions{}) + n, lastErr = w.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) if lastErr == nil { return true, nil } - log.Log.Error(lastErr, "getNodeState(): Failed to fetch node state, close all connections and retry...", "name", w.node) + log.Log.Error(lastErr, "getNodeState(): Failed to fetch node state, close all connections and retry...", "name", vars.NodeName) // Use the Get() also as an client-go keepalive indicator for the TCP connection. w.OnHeartbeatFailure() return false, nil }) if err != nil { if err == wait.ErrWaitTimeout { - return nil, errors.Wrapf(lastErr, "Timed out trying to fetch node %s", w.node) + return nil, errors.Wrapf(lastErr, "Timed out trying to fetch node %s", vars.NodeName) } return nil, err } return n, nil } -func (w *NodeStateStatusWriter) writeCheckpointFile(ns *sriovnetworkv1.SriovNetworkNodeState, destDir string) error { - configdir := filepath.Join(destDir, CheckpointFileName) +func (w *NodeStateStatusWriter) writeCheckpointFile(ns *sriovnetworkv1.SriovNetworkNodeState) error { + configdir := filepath.Join(vars.Destdir, CheckpointFileName) file, err := os.OpenFile(configdir, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return err } defer file.Close() log.Log.Info("writeCheckpointFile(): try to decode the checkpoint file") - if err = json.NewDecoder(file).Decode(&utils.InitialState); err != nil { + if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { log.Log.V(2).Error(err, "writeCheckpointFile(): fail to decode, writing new file instead") log.Log.Info("writeCheckpointFile(): write checkpoint file") if err = file.Truncate(0); err != nil { @@ -253,14 +245,14 @@ func (w *NodeStateStatusWriter) writeCheckpointFile(ns *sriovnetworkv1.SriovNetw if err = json.NewEncoder(file).Encode(*ns); err != nil { return err } - utils.InitialState = *ns + sriovnetworkv1.InitialState = *ns } return nil } -func (w *NodeStateStatusWriter) getCheckPointNodeState(destDir string) (*sriovnetworkv1.SriovNetworkNodeState, error) { +func (w *NodeStateStatusWriter) getCheckPointNodeState() (*sriovnetworkv1.SriovNetworkNodeState, error) { log.Log.Info("getCheckPointNodeState()") - configdir := filepath.Join(destDir, CheckpointFileName) + configdir := filepath.Join(vars.Destdir, CheckpointFileName) file, err := os.OpenFile(configdir, os.O_RDONLY, 0644) if err != nil { if os.IsNotExist(err) { @@ -269,9 +261,9 @@ func (w *NodeStateStatusWriter) getCheckPointNodeState(destDir string) (*sriovne return nil, err } defer file.Close() - if err = json.NewDecoder(file).Decode(&utils.InitialState); err != nil { + if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { return nil, err } - return &utils.InitialState, nil + return &sriovnetworkv1.InitialState, nil } diff --git a/pkg/helper/host.go b/pkg/helper/host.go new file mode 100644 index 000000000..df3325762 --- /dev/null +++ b/pkg/helper/host.go @@ -0,0 +1,49 @@ +package helper + +import ( + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" +) + +//go:generate ../../bin/mockgen -destination mock/mock_helper.go -source host.go +type HostHelpersInterface interface { + utils.CmdInterface + host.HostManagerInterface + host.StoreManagerInterface + mlx.MellanoxInterface +} + +type hostHelpers struct { + utils.CmdInterface + host.HostManagerInterface + host.StoreManagerInterface + mlx.MellanoxInterface +} + +// Use for unit tests +func NewHostHelpers(utilsHelper utils.CmdInterface, + hostManager host.HostManagerInterface, + storeManager host.StoreManagerInterface, + mlxHelper mlx.MellanoxInterface) HostHelpersInterface { + return &hostHelpers{utilsHelper, hostManager, storeManager, mlxHelper} +} + +func NewDefaultHostHelpers() (HostHelpersInterface, error) { + utilsHelper := utils.New() + mlxHelper := mlx.New(utilsHelper) + hostManager := host.NewHostManager(utilsHelper) + storeManager, err := host.NewStoreManager() + if err != nil { + log.Log.Error(err, "failed to create store manager") + return nil, err + } + + return &hostHelpers{ + utilsHelper, + hostManager, + storeManager, + mlxHelper}, nil +} diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go new file mode 100644 index 000000000..74814f4e6 --- /dev/null +++ b/pkg/helper/mock/mock_helper.go @@ -0,0 +1,1067 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: host.go + +// Package mock_helper is a generated GoMock package. +package mock_helper + +import ( + reflect "reflect" + + unit "github.com/coreos/go-systemd/v22/unit" + gomock "github.com/golang/mock/gomock" + ghw "github.com/jaypipes/ghw" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + host "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + mlxutils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" + netlink "github.com/vishvananda/netlink" +) + +// MockHostHelpersInterface is a mock of HostHelpersInterface interface. +type MockHostHelpersInterface struct { + ctrl *gomock.Controller + recorder *MockHostHelpersInterfaceMockRecorder +} + +// MockHostHelpersInterfaceMockRecorder is the mock recorder for MockHostHelpersInterface. +type MockHostHelpersInterfaceMockRecorder struct { + mock *MockHostHelpersInterface +} + +// NewMockHostHelpersInterface creates a new mock instance. +func NewMockHostHelpersInterface(ctrl *gomock.Controller) *MockHostHelpersInterface { + mock := &MockHostHelpersInterface{ctrl: ctrl} + mock.recorder = &MockHostHelpersInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockHostHelpersInterface) EXPECT() *MockHostHelpersInterfaceMockRecorder { + return m.recorder +} + +// AddUdevRule mocks base method. +func (m *MockHostHelpersInterface) AddUdevRule(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddUdevRule", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddUdevRule indicates an expected call of AddUdevRule. +func (mr *MockHostHelpersInterfaceMockRecorder) AddUdevRule(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddUdevRule), arg0) +} + +// BindDefaultDriver mocks base method. +func (m *MockHostHelpersInterface) BindDefaultDriver(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BindDefaultDriver", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// BindDefaultDriver indicates an expected call of BindDefaultDriver. +func (mr *MockHostHelpersInterfaceMockRecorder) BindDefaultDriver(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDefaultDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).BindDefaultDriver), arg0) +} + +// BindDpdkDriver mocks base method. +func (m *MockHostHelpersInterface) BindDpdkDriver(arg0, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BindDpdkDriver", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// BindDpdkDriver indicates an expected call of BindDpdkDriver. +func (mr *MockHostHelpersInterfaceMockRecorder) BindDpdkDriver(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDpdkDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).BindDpdkDriver), arg0, arg1) +} + +// Chroot mocks base method. +func (m *MockHostHelpersInterface) Chroot(arg0 string) (func() error, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Chroot", arg0) + ret0, _ := ret[0].(func() error) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Chroot indicates an expected call of Chroot. +func (mr *MockHostHelpersInterfaceMockRecorder) Chroot(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Chroot", reflect.TypeOf((*MockHostHelpersInterface)(nil).Chroot), arg0) +} + +// ClearPCIAddressFolder mocks base method. +func (m *MockHostHelpersInterface) ClearPCIAddressFolder() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearPCIAddressFolder") + ret0, _ := ret[0].(error) + return ret0 +} + +// ClearPCIAddressFolder indicates an expected call of ClearPCIAddressFolder. +func (mr *MockHostHelpersInterfaceMockRecorder) ClearPCIAddressFolder() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearPCIAddressFolder", reflect.TypeOf((*MockHostHelpersInterface)(nil).ClearPCIAddressFolder)) +} + +// CompareServices mocks base method. +func (m *MockHostHelpersInterface) CompareServices(serviceA, serviceB *host.Service) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompareServices", serviceA, serviceB) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompareServices indicates an expected call of CompareServices. +func (mr *MockHostHelpersInterfaceMockRecorder) CompareServices(serviceA, serviceB interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompareServices", reflect.TypeOf((*MockHostHelpersInterface)(nil).CompareServices), serviceA, serviceB) +} + +// ConfigSriovDevice mocks base method. +func (m *MockHostHelpersInterface) ConfigSriovDevice(iface *v1.Interface, ifaceStatus *v1.InterfaceExt) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigSriovDevice", iface, ifaceStatus) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigSriovDevice indicates an expected call of ConfigSriovDevice. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovDevice(iface, ifaceStatus interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovDevice), iface, ifaceStatus) +} + +// ConfigSriovDeviceVirtual mocks base method. +func (m *MockHostHelpersInterface) ConfigSriovDeviceVirtual(iface *v1.Interface) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigSriovDeviceVirtual", iface) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigSriovDeviceVirtual indicates an expected call of ConfigSriovDeviceVirtual. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovDeviceVirtual(iface interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovDeviceVirtual", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovDeviceVirtual), iface) +} + +// ConfigSriovInterfaces mocks base method. +func (m *MockHostHelpersInterface) ConfigSriovInterfaces(arg0 host.StoreManagerInterface, arg1 []v1.Interface, arg2 []v1.InterfaceExt, arg3 map[string]bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigSriovInterfaces", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigSriovInterfaces indicates an expected call of ConfigSriovInterfaces. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), arg0, arg1, arg2, arg3) +} + +// DiscoverSriovDevices mocks base method. +func (m *MockHostHelpersInterface) DiscoverSriovDevices(arg0 host.StoreManagerInterface) ([]v1.InterfaceExt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverSriovDevices", arg0) + ret0, _ := ret[0].([]v1.InterfaceExt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiscoverSriovDevices indicates an expected call of DiscoverSriovDevices. +func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverSriovDevices(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevices", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverSriovDevices), arg0) +} + +// EnableRDMA mocks base method. +func (m *MockHostHelpersInterface) EnableRDMA(arg0, arg1, arg2 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnableRDMA", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnableRDMA indicates an expected call of EnableRDMA. +func (mr *MockHostHelpersInterfaceMockRecorder) EnableRDMA(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableRDMA", reflect.TypeOf((*MockHostHelpersInterface)(nil).EnableRDMA), arg0, arg1, arg2) +} + +// EnableRDMAOnRHELMachine mocks base method. +func (m *MockHostHelpersInterface) EnableRDMAOnRHELMachine() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnableRDMAOnRHELMachine") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnableRDMAOnRHELMachine indicates an expected call of EnableRDMAOnRHELMachine. +func (mr *MockHostHelpersInterfaceMockRecorder) EnableRDMAOnRHELMachine() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableRDMAOnRHELMachine", reflect.TypeOf((*MockHostHelpersInterface)(nil).EnableRDMAOnRHELMachine)) +} + +// EnableService mocks base method. +func (m *MockHostHelpersInterface) EnableService(service *host.Service) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnableService", service) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnableService indicates an expected call of EnableService. +func (mr *MockHostHelpersInterfaceMockRecorder) EnableService(service interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableService", reflect.TypeOf((*MockHostHelpersInterface)(nil).EnableService), service) +} + +// GetCheckPointNodeState mocks base method. +func (m *MockHostHelpersInterface) GetCheckPointNodeState() (*v1.SriovNetworkNodeState, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCheckPointNodeState") + ret0, _ := ret[0].(*v1.SriovNetworkNodeState) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCheckPointNodeState indicates an expected call of GetCheckPointNodeState. +func (mr *MockHostHelpersInterfaceMockRecorder) GetCheckPointNodeState() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCheckPointNodeState", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetCheckPointNodeState)) +} + +// GetCurrentKernelArgs mocks base method. +func (m *MockHostHelpersInterface) GetCurrentKernelArgs() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentKernelArgs") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCurrentKernelArgs indicates an expected call of GetCurrentKernelArgs. +func (mr *MockHostHelpersInterfaceMockRecorder) GetCurrentKernelArgs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentKernelArgs", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetCurrentKernelArgs)) +} + +// GetLinkType mocks base method. +func (m *MockHostHelpersInterface) GetLinkType(arg0 v1.InterfaceExt) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLinkType", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetLinkType indicates an expected call of GetLinkType. +func (mr *MockHostHelpersInterfaceMockRecorder) GetLinkType(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkType", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetLinkType), arg0) +} + +// GetMellanoxBlueFieldMode mocks base method. +func (m *MockHostHelpersInterface) GetMellanoxBlueFieldMode(arg0 string) (mlxutils.BlueFieldMode, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMellanoxBlueFieldMode", arg0) + ret0, _ := ret[0].(mlxutils.BlueFieldMode) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMellanoxBlueFieldMode indicates an expected call of GetMellanoxBlueFieldMode. +func (mr *MockHostHelpersInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMellanoxBlueFieldMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetMellanoxBlueFieldMode), arg0) +} + +// GetMlxNicFwData mocks base method. +func (m *MockHostHelpersInterface) GetMlxNicFwData(pciAddress string) (*mlxutils.MlxNic, *mlxutils.MlxNic, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMlxNicFwData", pciAddress) + ret0, _ := ret[0].(*mlxutils.MlxNic) + ret1, _ := ret[1].(*mlxutils.MlxNic) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetMlxNicFwData indicates an expected call of GetMlxNicFwData. +func (mr *MockHostHelpersInterfaceMockRecorder) GetMlxNicFwData(pciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMlxNicFwData", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetMlxNicFwData), pciAddress) +} + +// GetNetDevLinkSpeed mocks base method. +func (m *MockHostHelpersInterface) GetNetDevLinkSpeed(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNetDevLinkSpeed", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetNetDevLinkSpeed indicates an expected call of GetNetDevLinkSpeed. +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevLinkSpeed(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevLinkSpeed", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevLinkSpeed), arg0) +} + +// GetNetDevMac mocks base method. +func (m *MockHostHelpersInterface) GetNetDevMac(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNetDevMac", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetNetDevMac indicates an expected call of GetNetDevMac. +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevMac(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevMac", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevMac), arg0) +} + +// GetNetdevMTU mocks base method. +func (m *MockHostHelpersInterface) GetNetdevMTU(arg0 string) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNetdevMTU", arg0) + ret0, _ := ret[0].(int) + return ret0 +} + +// GetNetdevMTU indicates an expected call of GetNetdevMTU. +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetdevMTU(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetdevMTU", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetdevMTU), arg0) +} + +// GetNicSriovMode mocks base method. +func (m *MockHostHelpersInterface) GetNicSriovMode(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNicSriovMode", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNicSriovMode indicates an expected call of GetNicSriovMode. +func (mr *MockHostHelpersInterfaceMockRecorder) GetNicSriovMode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNicSriovMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNicSriovMode), arg0) +} + +// GetOSPrettyName mocks base method. +func (m *MockHostHelpersInterface) GetOSPrettyName() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOSPrettyName") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetOSPrettyName indicates an expected call of GetOSPrettyName. +func (mr *MockHostHelpersInterfaceMockRecorder) GetOSPrettyName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOSPrettyName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetOSPrettyName)) +} + +// GetPhysPortName mocks base method. +func (m *MockHostHelpersInterface) GetPhysPortName(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPhysPortName", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPhysPortName indicates an expected call of GetPhysPortName. +func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysPortName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysPortName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPhysPortName), arg0) +} + +// GetPhysSwitchID mocks base method. +func (m *MockHostHelpersInterface) GetPhysSwitchID(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPhysSwitchID", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPhysSwitchID indicates an expected call of GetPhysSwitchID. +func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysSwitchID(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysSwitchID", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPhysSwitchID), arg0) +} + +// GetVfInfo mocks base method. +func (m *MockHostHelpersInterface) GetVfInfo(arg0 string, arg1 []*ghw.PCIDevice) v1.VirtualFunction { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVfInfo", arg0, arg1) + ret0, _ := ret[0].(v1.VirtualFunction) + return ret0 +} + +// GetVfInfo indicates an expected call of GetVfInfo. +func (mr *MockHostHelpersInterfaceMockRecorder) GetVfInfo(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfInfo", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetVfInfo), arg0, arg1) +} + +// HasDriver mocks base method. +func (m *MockHostHelpersInterface) HasDriver(arg0 string) (bool, string) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasDriver", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + return ret0, ret1 +} + +// HasDriver indicates an expected call of HasDriver. +func (mr *MockHostHelpersInterfaceMockRecorder) HasDriver(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).HasDriver), arg0) +} + +// InstallRDMA mocks base method. +func (m *MockHostHelpersInterface) InstallRDMA(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallRDMA", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallRDMA indicates an expected call of InstallRDMA. +func (mr *MockHostHelpersInterfaceMockRecorder) InstallRDMA(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallRDMA", reflect.TypeOf((*MockHostHelpersInterface)(nil).InstallRDMA), arg0) +} + +// IsCoreOS mocks base method. +func (m *MockHostHelpersInterface) IsCoreOS() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCoreOS") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsCoreOS indicates an expected call of IsCoreOS. +func (mr *MockHostHelpersInterfaceMockRecorder) IsCoreOS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCoreOS", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsCoreOS)) +} + +// IsKernelArgsSet mocks base method. +func (m *MockHostHelpersInterface) IsKernelArgsSet(arg0, arg1 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsKernelArgsSet", arg0, arg1) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsKernelArgsSet indicates an expected call of IsKernelArgsSet. +func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelArgsSet(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelArgsSet", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsKernelArgsSet), arg0, arg1) +} + +// IsKernelLockdownMode mocks base method. +func (m *MockHostHelpersInterface) IsKernelLockdownMode() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsKernelLockdownMode") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsKernelLockdownMode indicates an expected call of IsKernelLockdownMode. +func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelLockdownMode() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelLockdownMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsKernelLockdownMode)) +} + +// IsKernelModuleLoaded mocks base method. +func (m *MockHostHelpersInterface) IsKernelModuleLoaded(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsKernelModuleLoaded", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsKernelModuleLoaded indicates an expected call of IsKernelModuleLoaded. +func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelModuleLoaded(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelModuleLoaded", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsKernelModuleLoaded), arg0) +} + +// IsRHELSystem mocks base method. +func (m *MockHostHelpersInterface) IsRHELSystem() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsRHELSystem") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsRHELSystem indicates an expected call of IsRHELSystem. +func (mr *MockHostHelpersInterfaceMockRecorder) IsRHELSystem() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRHELSystem", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsRHELSystem)) +} + +// IsServiceEnabled mocks base method. +func (m *MockHostHelpersInterface) IsServiceEnabled(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsServiceEnabled", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsServiceEnabled indicates an expected call of IsServiceEnabled. +func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceEnabled(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceEnabled", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsServiceEnabled), arg0) +} + +// IsServiceExist mocks base method. +func (m *MockHostHelpersInterface) IsServiceExist(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsServiceExist", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsServiceExist indicates an expected call of IsServiceExist. +func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceExist(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceExist", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsServiceExist), arg0) +} + +// IsSwitchdev mocks base method. +func (m *MockHostHelpersInterface) IsSwitchdev(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsSwitchdev", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsSwitchdev indicates an expected call of IsSwitchdev. +func (mr *MockHostHelpersInterfaceMockRecorder) IsSwitchdev(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSwitchdev", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsSwitchdev), arg0) +} + +// IsUbuntuSystem mocks base method. +func (m *MockHostHelpersInterface) IsUbuntuSystem() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsUbuntuSystem") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsUbuntuSystem indicates an expected call of IsUbuntuSystem. +func (mr *MockHostHelpersInterfaceMockRecorder) IsUbuntuSystem() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUbuntuSystem", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsUbuntuSystem)) +} + +// LoadKernelModule mocks base method. +func (m *MockHostHelpersInterface) LoadKernelModule(name string, args ...string) error { + m.ctrl.T.Helper() + varargs := []interface{}{name} + for _, a := range args { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "LoadKernelModule", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadKernelModule indicates an expected call of LoadKernelModule. +func (mr *MockHostHelpersInterfaceMockRecorder) LoadKernelModule(name interface{}, args ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{name}, args...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadKernelModule", reflect.TypeOf((*MockHostHelpersInterface)(nil).LoadKernelModule), varargs...) +} + +// LoadPfsStatus mocks base method. +func (m *MockHostHelpersInterface) LoadPfsStatus(pciAddress string) (*v1.Interface, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadPfsStatus", pciAddress) + ret0, _ := ret[0].(*v1.Interface) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// LoadPfsStatus indicates an expected call of LoadPfsStatus. +func (mr *MockHostHelpersInterfaceMockRecorder) LoadPfsStatus(pciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPfsStatus", reflect.TypeOf((*MockHostHelpersInterface)(nil).LoadPfsStatus), pciAddress) +} + +// MlxConfigFW mocks base method. +func (m *MockHostHelpersInterface) MlxConfigFW(attributesToChange map[string]mlxutils.MlxNic) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MlxConfigFW", attributesToChange) + ret0, _ := ret[0].(error) + return ret0 +} + +// MlxConfigFW indicates an expected call of MlxConfigFW. +func (mr *MockHostHelpersInterfaceMockRecorder) MlxConfigFW(attributesToChange interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxConfigFW", reflect.TypeOf((*MockHostHelpersInterface)(nil).MlxConfigFW), attributesToChange) +} + +// MstConfigReadData mocks base method. +func (m *MockHostHelpersInterface) MstConfigReadData(arg0 string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MstConfigReadData", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// MstConfigReadData indicates an expected call of MstConfigReadData. +func (mr *MockHostHelpersInterfaceMockRecorder) MstConfigReadData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MstConfigReadData", reflect.TypeOf((*MockHostHelpersInterface)(nil).MstConfigReadData), arg0) +} + +// PrepareNMUdevRule mocks base method. +func (m *MockHostHelpersInterface) PrepareNMUdevRule(arg0 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PrepareNMUdevRule", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// PrepareNMUdevRule indicates an expected call of PrepareNMUdevRule. +func (mr *MockHostHelpersInterfaceMockRecorder) PrepareNMUdevRule(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareNMUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).PrepareNMUdevRule), arg0) +} + +// RdmaIsLoaded mocks base method. +func (m *MockHostHelpersInterface) RdmaIsLoaded() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RdmaIsLoaded") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RdmaIsLoaded indicates an expected call of RdmaIsLoaded. +func (mr *MockHostHelpersInterfaceMockRecorder) RdmaIsLoaded() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaIsLoaded", reflect.TypeOf((*MockHostHelpersInterface)(nil).RdmaIsLoaded)) +} + +// ReadScriptManifestFile mocks base method. +func (m *MockHostHelpersInterface) ReadScriptManifestFile(path string) (*host.ScriptManifestFile, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadScriptManifestFile", path) + ret0, _ := ret[0].(*host.ScriptManifestFile) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadScriptManifestFile indicates an expected call of ReadScriptManifestFile. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadScriptManifestFile(path interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadScriptManifestFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadScriptManifestFile), path) +} + +// ReadService mocks base method. +func (m *MockHostHelpersInterface) ReadService(arg0 string) (*host.Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadService", arg0) + ret0, _ := ret[0].(*host.Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadService indicates an expected call of ReadService. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadService(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadService", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadService), arg0) +} + +// ReadServiceInjectionManifestFile mocks base method. +func (m *MockHostHelpersInterface) ReadServiceInjectionManifestFile(path string) (*host.Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadServiceInjectionManifestFile", path) + ret0, _ := ret[0].(*host.Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadServiceInjectionManifestFile indicates an expected call of ReadServiceInjectionManifestFile. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceInjectionManifestFile(path interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceInjectionManifestFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadServiceInjectionManifestFile), path) +} + +// ReadServiceManifestFile mocks base method. +func (m *MockHostHelpersInterface) ReadServiceManifestFile(path string) (*host.Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadServiceManifestFile", path) + ret0, _ := ret[0].(*host.Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadServiceManifestFile indicates an expected call of ReadServiceManifestFile. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceManifestFile(path interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceManifestFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadServiceManifestFile), path) +} + +// RebindVfToDefaultDriver mocks base method. +func (m *MockHostHelpersInterface) RebindVfToDefaultDriver(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RebindVfToDefaultDriver", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RebindVfToDefaultDriver indicates an expected call of RebindVfToDefaultDriver. +func (mr *MockHostHelpersInterfaceMockRecorder) RebindVfToDefaultDriver(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebindVfToDefaultDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).RebindVfToDefaultDriver), arg0) +} + +// ReloadDriver mocks base method. +func (m *MockHostHelpersInterface) ReloadDriver(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReloadDriver", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReloadDriver indicates an expected call of ReloadDriver. +func (mr *MockHostHelpersInterfaceMockRecorder) ReloadDriver(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReloadDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReloadDriver), arg0) +} + +// RemoveFromService mocks base method. +func (m *MockHostHelpersInterface) RemoveFromService(service *host.Service, options ...*unit.UnitOption) (*host.Service, error) { + m.ctrl.T.Helper() + varargs := []interface{}{service} + for _, a := range options { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RemoveFromService", varargs...) + ret0, _ := ret[0].(*host.Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFromService indicates an expected call of RemoveFromService. +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveFromService(service interface{}, options ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{service}, options...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFromService", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveFromService), varargs...) +} + +// RemoveUdevRule mocks base method. +func (m *MockHostHelpersInterface) RemoveUdevRule(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveUdevRule", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveUdevRule indicates an expected call of RemoveUdevRule. +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveUdevRule(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveUdevRule), arg0) +} + +// ResetSriovDevice mocks base method. +func (m *MockHostHelpersInterface) ResetSriovDevice(arg0 v1.InterfaceExt) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResetSriovDevice", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ResetSriovDevice indicates an expected call of ResetSriovDevice. +func (mr *MockHostHelpersInterfaceMockRecorder) ResetSriovDevice(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetSriovDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).ResetSriovDevice), arg0) +} + +// RunCommand mocks base method. +func (m *MockHostHelpersInterface) RunCommand(arg0 string, arg1 ...string) (string, string, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RunCommand", varargs...) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// RunCommand indicates an expected call of RunCommand. +func (mr *MockHostHelpersInterfaceMockRecorder) RunCommand(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunCommand", reflect.TypeOf((*MockHostHelpersInterface)(nil).RunCommand), varargs...) +} + +// SaveLastPfAppliedStatus mocks base method. +func (m *MockHostHelpersInterface) SaveLastPfAppliedStatus(PfInfo *v1.Interface) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveLastPfAppliedStatus", PfInfo) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveLastPfAppliedStatus indicates an expected call of SaveLastPfAppliedStatus. +func (mr *MockHostHelpersInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveLastPfAppliedStatus", reflect.TypeOf((*MockHostHelpersInterface)(nil).SaveLastPfAppliedStatus), PfInfo) +} + +// SetNetdevMTU mocks base method. +func (m *MockHostHelpersInterface) SetNetdevMTU(arg0 string, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetNetdevMTU", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetNetdevMTU indicates an expected call of SetNetdevMTU. +func (mr *MockHostHelpersInterfaceMockRecorder) SetNetdevMTU(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetdevMTU", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetNetdevMTU), arg0, arg1) +} + +// SetSriovNumVfs mocks base method. +func (m *MockHostHelpersInterface) SetSriovNumVfs(arg0 string, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetSriovNumVfs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetSriovNumVfs indicates an expected call of SetSriovNumVfs. +func (mr *MockHostHelpersInterfaceMockRecorder) SetSriovNumVfs(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSriovNumVfs", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetSriovNumVfs), arg0, arg1) +} + +// SetVfAdminMac mocks base method. +func (m *MockHostHelpersInterface) SetVfAdminMac(arg0 string, arg1, arg2 netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetVfAdminMac", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetVfAdminMac indicates an expected call of SetVfAdminMac. +func (mr *MockHostHelpersInterfaceMockRecorder) SetVfAdminMac(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfAdminMac", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetVfAdminMac), arg0, arg1, arg2) +} + +// SetVfGUID mocks base method. +func (m *MockHostHelpersInterface) SetVfGUID(arg0 string, arg1 netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetVfGUID", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetVfGUID indicates an expected call of SetVfGUID. +func (mr *MockHostHelpersInterfaceMockRecorder) SetVfGUID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetVfGUID), arg0, arg1) +} + +// TriggerUdevEvent mocks base method. +func (m *MockHostHelpersInterface) TriggerUdevEvent() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerUdevEvent") + ret0, _ := ret[0].(error) + return ret0 +} + +// TriggerUdevEvent indicates an expected call of TriggerUdevEvent. +func (mr *MockHostHelpersInterfaceMockRecorder) TriggerUdevEvent() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerUdevEvent", reflect.TypeOf((*MockHostHelpersInterface)(nil).TriggerUdevEvent)) +} + +// TryEnableRdma mocks base method. +func (m *MockHostHelpersInterface) TryEnableRdma() (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryEnableRdma") + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TryEnableRdma indicates an expected call of TryEnableRdma. +func (mr *MockHostHelpersInterfaceMockRecorder) TryEnableRdma() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableRdma", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryEnableRdma)) +} + +// TryEnableTun mocks base method. +func (m *MockHostHelpersInterface) TryEnableTun() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "TryEnableTun") +} + +// TryEnableTun indicates an expected call of TryEnableTun. +func (mr *MockHostHelpersInterfaceMockRecorder) TryEnableTun() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableTun", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryEnableTun)) +} + +// TryEnableVhostNet mocks base method. +func (m *MockHostHelpersInterface) TryEnableVhostNet() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "TryEnableVhostNet") +} + +// TryEnableVhostNet indicates an expected call of TryEnableVhostNet. +func (mr *MockHostHelpersInterfaceMockRecorder) TryEnableVhostNet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableVhostNet", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryEnableVhostNet)) +} + +// TryGetInterfaceName mocks base method. +func (m *MockHostHelpersInterface) TryGetInterfaceName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryGetInterfaceName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// TryGetInterfaceName indicates an expected call of TryGetInterfaceName. +func (mr *MockHostHelpersInterfaceMockRecorder) TryGetInterfaceName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryGetInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryGetInterfaceName), arg0) +} + +// TryToGetVirtualInterfaceName mocks base method. +func (m *MockHostHelpersInterface) TryToGetVirtualInterfaceName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryToGetVirtualInterfaceName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// TryToGetVirtualInterfaceName indicates an expected call of TryToGetVirtualInterfaceName. +func (mr *MockHostHelpersInterfaceMockRecorder) TryToGetVirtualInterfaceName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryToGetVirtualInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryToGetVirtualInterfaceName), arg0) +} + +// Unbind mocks base method. +func (m *MockHostHelpersInterface) Unbind(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Unbind", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Unbind indicates an expected call of Unbind. +func (mr *MockHostHelpersInterfaceMockRecorder) Unbind(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unbind", reflect.TypeOf((*MockHostHelpersInterface)(nil).Unbind), arg0) +} + +// UnbindDriverIfNeeded mocks base method. +func (m *MockHostHelpersInterface) UnbindDriverIfNeeded(arg0 string, arg1 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnbindDriverIfNeeded", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UnbindDriverIfNeeded indicates an expected call of UnbindDriverIfNeeded. +func (mr *MockHostHelpersInterfaceMockRecorder) UnbindDriverIfNeeded(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnbindDriverIfNeeded", reflect.TypeOf((*MockHostHelpersInterface)(nil).UnbindDriverIfNeeded), arg0, arg1) +} + +// UpdateSystemService mocks base method. +func (m *MockHostHelpersInterface) UpdateSystemService(serviceObj *host.Service) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateSystemService", serviceObj) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateSystemService indicates an expected call of UpdateSystemService. +func (mr *MockHostHelpersInterfaceMockRecorder) UpdateSystemService(serviceObj interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSystemService", reflect.TypeOf((*MockHostHelpersInterface)(nil).UpdateSystemService), serviceObj) +} + +// VFIsReady mocks base method. +func (m *MockHostHelpersInterface) VFIsReady(arg0 string) (netlink.Link, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VFIsReady", arg0) + ret0, _ := ret[0].(netlink.Link) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VFIsReady indicates an expected call of VFIsReady. +func (mr *MockHostHelpersInterfaceMockRecorder) VFIsReady(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VFIsReady", reflect.TypeOf((*MockHostHelpersInterface)(nil).VFIsReady), arg0) +} + +// WriteCheckpointFile mocks base method. +func (m *MockHostHelpersInterface) WriteCheckpointFile(arg0 *v1.SriovNetworkNodeState) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteCheckpointFile", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteCheckpointFile indicates an expected call of WriteCheckpointFile. +func (mr *MockHostHelpersInterfaceMockRecorder) WriteCheckpointFile(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteCheckpointFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteCheckpointFile), arg0) +} + +// WriteSwitchdevConfFile mocks base method. +func (m *MockHostHelpersInterface) WriteSwitchdevConfFile(arg0 *v1.SriovNetworkNodeState, arg1 map[string]bool) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteSwitchdevConfFile", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteSwitchdevConfFile indicates an expected call of WriteSwitchdevConfFile. +func (mr *MockHostHelpersInterfaceMockRecorder) WriteSwitchdevConfFile(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteSwitchdevConfFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteSwitchdevConfFile), arg0, arg1) +} diff --git a/pkg/host/host.go b/pkg/host/host.go deleted file mode 100644 index 241e6ba17..000000000 --- a/pkg/host/host.go +++ /dev/null @@ -1,457 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package host - -import ( - "fmt" - "os" - pathlib "path" - "strings" - - "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -const ( - hostPathFromDaemon = "/host" - redhatReleaseFile = "/etc/redhat-release" - rhelRDMAConditionFile = "/usr/libexec/rdma-init-kernel" - rhelRDMAServiceName = "rdma" - rhelPackageManager = "yum" - - ubuntuRDMAConditionFile = "/usr/sbin/rdma-ndd" - ubuntuRDMAServiceName = "rdma-ndd" - ubuntuPackageManager = "apt-get" - - genericOSReleaseFile = "/etc/os-release" -) - -// Contains all the host manipulation functions -// -//go:generate ../../bin/mockgen -destination mock/mock_host.go -source host.go -type HostManagerInterface interface { - TryEnableTun() - TryEnableVhostNet() - TryEnableRdma() (bool, error) - - // private functions - // part of the interface for the mock generation - LoadKernelModule(name string, args ...string) error - IsKernelModuleLoaded(string) (bool, error) - IsRHELSystem() (bool, error) - IsUbuntuSystem() (bool, error) - IsCoreOS() (bool, error) - RdmaIsLoaded() (bool, error) - EnableRDMA(string, string, string) (bool, error) - InstallRDMA(string) error - TriggerUdevEvent() error - ReloadDriver(string) error - EnableRDMAOnRHELMachine() (bool, error) - GetOSPrettyName() (string, error) -} - -type HostManager struct { - RunOnHost bool - cmd utils.CommandInterface -} - -func NewHostManager(runOnHost bool) HostManagerInterface { - return &HostManager{ - RunOnHost: runOnHost, - cmd: &utils.Command{}, - } -} - -func (h *HostManager) LoadKernelModule(name string, args ...string) error { - log.Log.Info("LoadKernelModule(): try to load kernel module", "name", name, "args", args) - chrootDefinition := getChrootExtension(h.RunOnHost) - cmdArgs := strings.Join(args, " ") - - // check if the driver is already loaded in to the system - isLoaded, err := h.IsKernelModuleLoaded(name) - if err != nil { - log.Log.Error(err, "LoadKernelModule(): failed to check if kernel module is already loaded", "name", name) - } - if isLoaded { - log.Log.Info("LoadKernelModule(): kernel module already loaded", "name", name) - return nil - } - - _, _, err = h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("%s modprobe %s %s", chrootDefinition, name, cmdArgs)) - if err != nil { - log.Log.Error(err, "LoadKernelModule(): failed to load kernel module with arguments", "name", name, "args", args) - return err - } - return nil -} - -func (h *HostManager) IsKernelModuleLoaded(kernelModuleName string) (bool, error) { - log.Log.Info("IsKernelModuleLoaded(): check if kernel module is loaded", "name", kernelModuleName) - chrootDefinition := getChrootExtension(h.RunOnHost) - - stdout, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("%s lsmod | grep \"^%s\"", chrootDefinition, kernelModuleName)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", - "name", kernelModuleName, "stderr", stderr.String()) - return false, err - } - log.Log.V(2).Info("IsKernelModuleLoaded():", "stdout", stdout.String()) - if stderr.Len() != 0 { - log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", "name", kernelModuleName, "stderr", stderr.String()) - return false, fmt.Errorf(stderr.String()) - } - - if stdout.Len() != 0 { - log.Log.Info("IsKernelModuleLoaded(): kernel module already loaded", "name", kernelModuleName) - return true, nil - } - - return false, nil -} - -func (h *HostManager) TryEnableTun() { - if err := h.LoadKernelModule("tun"); err != nil { - log.Log.Error(err, "tryEnableTun(): TUN kernel module not loaded") - } -} - -func (h *HostManager) TryEnableVhostNet() { - if err := h.LoadKernelModule("vhost_net"); err != nil { - log.Log.Error(err, "tryEnableVhostNet(): VHOST_NET kernel module not loaded") - } -} - -func (h *HostManager) TryEnableRdma() (bool, error) { - log.Log.V(2).Info("tryEnableRdma()") - chrootDefinition := getChrootExtension(h.RunOnHost) - - // check if the driver is already loaded in to the system - _, stderr, mlx4Err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("grep --quiet 'mlx4_en' <(%s lsmod)", chrootDefinition)) - if mlx4Err != nil && stderr.Len() != 0 { - log.Log.Error(mlx4Err, "tryEnableRdma(): failed to check for kernel module 'mlx4_en'", "stderr", stderr.String()) - return false, fmt.Errorf(stderr.String()) - } - - _, stderr, mlx5Err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("grep --quiet 'mlx5_core' <(%s lsmod)", chrootDefinition)) - if mlx5Err != nil && stderr.Len() != 0 { - log.Log.Error(mlx5Err, "tryEnableRdma(): failed to check for kernel module 'mlx5_core'", "stderr", stderr.String()) - return false, fmt.Errorf(stderr.String()) - } - - if mlx4Err != nil && mlx5Err != nil { - log.Log.Error(nil, "tryEnableRdma(): no RDMA capable devices") - return false, nil - } - - isRhelSystem, err := h.IsRHELSystem() - if err != nil { - log.Log.Error(err, "tryEnableRdma(): failed to check if the machine is base on RHEL") - return false, err - } - - // RHEL check - if isRhelSystem { - return h.EnableRDMAOnRHELMachine() - } - - isUbuntuSystem, err := h.IsUbuntuSystem() - if err != nil { - log.Log.Error(err, "tryEnableRdma(): failed to check if the machine is base on Ubuntu") - return false, err - } - - if isUbuntuSystem { - return h.EnableRDMAOnUbuntuMachine() - } - - osName, err := h.GetOSPrettyName() - if err != nil { - log.Log.Error(err, "tryEnableRdma(): failed to check OS name") - return false, err - } - - log.Log.Error(nil, "tryEnableRdma(): Unsupported OS", "name", osName) - return false, fmt.Errorf("unable to load RDMA unsupported OS: %s", osName) -} - -func (h *HostManager) EnableRDMAOnRHELMachine() (bool, error) { - log.Log.Info("EnableRDMAOnRHELMachine()") - isCoreOsSystem, err := h.IsCoreOS() - if err != nil { - log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if the machine runs CoreOS") - return false, err - } - - // CoreOS check - if isCoreOsSystem { - isRDMALoaded, err := h.RdmaIsLoaded() - if err != nil { - log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if RDMA kernel modules are loaded") - return false, err - } - - return isRDMALoaded, nil - } - - // RHEL - log.Log.Info("EnableRDMAOnRHELMachine(): enabling RDMA on RHEL machine") - isRDMAEnable, err := h.EnableRDMA(rhelRDMAConditionFile, rhelRDMAServiceName, rhelPackageManager) - if err != nil { - log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to enable RDMA on RHEL machine") - return false, err - } - - // check if we need to install rdma-core package - if isRDMAEnable { - isRDMALoaded, err := h.RdmaIsLoaded() - if err != nil { - log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if RDMA kernel modules are loaded") - return false, err - } - - // if ib kernel module is not loaded trigger a loading - if isRDMALoaded { - err = h.TriggerUdevEvent() - if err != nil { - log.Log.Error(err, "EnableRDMAOnRHELMachine() failed to trigger udev event") - return false, err - } - } - } - - return true, nil -} - -func (h *HostManager) EnableRDMAOnUbuntuMachine() (bool, error) { - log.Log.Info("EnableRDMAOnUbuntuMachine(): enabling RDMA on RHEL machine") - isRDMAEnable, err := h.EnableRDMA(ubuntuRDMAConditionFile, ubuntuRDMAServiceName, ubuntuPackageManager) - if err != nil { - log.Log.Error(err, "EnableRDMAOnUbuntuMachine(): failed to enable RDMA on Ubuntu machine") - return false, err - } - - // check if we need to install rdma-core package - if isRDMAEnable { - isRDMALoaded, err := h.RdmaIsLoaded() - if err != nil { - log.Log.Error(err, "EnableRDMAOnUbuntuMachine(): failed to check if RDMA kernel modules are loaded") - return false, err - } - - // if ib kernel module is not loaded trigger a loading - if isRDMALoaded { - err = h.TriggerUdevEvent() - if err != nil { - log.Log.Error(err, "EnableRDMAOnUbuntuMachine() failed to trigger udev event") - return false, err - } - } - } - - return true, nil -} - -func (h *HostManager) IsRHELSystem() (bool, error) { - log.Log.Info("IsRHELSystem(): checking for RHEL machine") - path := redhatReleaseFile - if !h.RunOnHost { - path = pathlib.Join(hostPathFromDaemon, path) - } - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("IsRHELSystem() not a RHEL machine") - return false, nil - } - - log.Log.Error(err, "IsRHELSystem() failed to check for os release file", "path", path) - return false, err - } - - return true, nil -} - -func (h *HostManager) IsCoreOS() (bool, error) { - log.Log.Info("IsCoreOS(): checking for CoreOS machine") - path := redhatReleaseFile - if !h.RunOnHost { - path = pathlib.Join(hostPathFromDaemon, path) - } - - data, err := os.ReadFile(path) - if err != nil { - log.Log.Error(err, "IsCoreOS(): failed to read RHEL release file on path", "path", path) - return false, err - } - - if strings.Contains(string(data), "CoreOS") { - return true, nil - } - - return false, nil -} - -func (h *HostManager) IsUbuntuSystem() (bool, error) { - log.Log.Info("IsUbuntuSystem(): checking for Ubuntu machine") - path := genericOSReleaseFile - if !h.RunOnHost { - path = pathlib.Join(hostPathFromDaemon, path) - } - - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - log.Log.Error(nil, "IsUbuntuSystem() os-release on path doesn't exist", "path", path) - return false, err - } - - log.Log.Error(err, "IsUbuntuSystem() failed to check for os release file", "path", path) - return false, err - } - - stdout, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("grep -i --quiet 'ubuntu' %s", path)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "IsUbuntuSystem(): failed to check for ubuntu operating system name in os-releasae file", "stderr", stderr.String()) - return false, fmt.Errorf(stderr.String()) - } - - if stdout.Len() > 0 { - return true, nil - } - - return false, nil -} - -func (h *HostManager) RdmaIsLoaded() (bool, error) { - log.Log.V(2).Info("RdmaIsLoaded()") - chrootDefinition := getChrootExtension(h.RunOnHost) - - // check if the driver is already loaded in to the system - _, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("grep --quiet '\\(^ib\\|^rdma\\)' <(%s lsmod)", chrootDefinition)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "RdmaIsLoaded(): fail to check if ib and rdma kernel modules are loaded", "stderr", stderr.String()) - return false, fmt.Errorf(stderr.String()) - } - - if err != nil { - return false, nil - } - - return true, nil -} - -func (h *HostManager) EnableRDMA(conditionFilePath, serviceName, packageManager string) (bool, error) { - path := conditionFilePath - if !h.RunOnHost { - path = pathlib.Join(hostPathFromDaemon, path) - } - log.Log.Info("EnableRDMA(): checking for service file", "path", path) - - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("EnableRDMA(): RDMA server doesn't exist") - err = h.InstallRDMA(packageManager) - if err != nil { - log.Log.Error(err, "EnableRDMA() failed to install RDMA package") - return false, err - } - - err = h.TriggerUdevEvent() - if err != nil { - log.Log.Error(err, "EnableRDMA() failed to trigger udev event") - return false, err - } - - return false, nil - } - - log.Log.Error(err, "EnableRDMA() failed to check for os release file", "path", path) - return false, err - } - - log.Log.Info("EnableRDMA(): service installed", "name", serviceName) - return true, nil -} - -func (h *HostManager) InstallRDMA(packageManager string) error { - log.Log.Info("InstallRDMA(): installing RDMA") - chrootDefinition := getChrootExtension(h.RunOnHost) - - stdout, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("%s %s install -y rdma-core", chrootDefinition, packageManager)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "InstallRDMA(): failed to install RDMA package", "stdout", stdout.String(), "stderr", stderr.String()) - return err - } - - return nil -} - -func (h *HostManager) TriggerUdevEvent() error { - log.Log.Info("TriggerUdevEvent(): installing RDMA") - - err := h.ReloadDriver("mlx4_en") - if err != nil { - return err - } - - err = h.ReloadDriver("mlx5_core") - if err != nil { - return err - } - - return nil -} - -func (h *HostManager) ReloadDriver(driverName string) error { - log.Log.Info("ReloadDriver(): reload driver", "name", driverName) - chrootDefinition := getChrootExtension(h.RunOnHost) - - _, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("%s modprobe -r %s && %s modprobe %s", chrootDefinition, driverName, chrootDefinition, driverName)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "InstallRDMA(): failed to reload kernel module", - "name", driverName, "stderr", stderr.String()) - return err - } - - return nil -} - -func (h *HostManager) GetOSPrettyName() (string, error) { - path := genericOSReleaseFile - if !h.RunOnHost { - path = pathlib.Join(hostPathFromDaemon, path) - } - - log.Log.Info("GetOSPrettyName(): getting os name from os-release file") - - stdout, stderr, err := h.cmd.Run("/bin/sh", "-c", fmt.Sprintf("cat %s | grep PRETTY_NAME | cut -c 13-", path)) - if err != nil && stderr.Len() != 0 { - log.Log.Error(err, "IsUbuntuSystem(): failed to check for ubuntu operating system name in os-releasae file", "stderr", stderr.String()) - return "", fmt.Errorf(stderr.String()) - } - - if stdout.Len() > 0 { - return stdout.String(), nil - } - - return "", fmt.Errorf("failed to find pretty operating system name") -} - -func getChrootExtension(runOnHost bool) string { - if !runOnHost { - return fmt.Sprintf("chroot %s/host", utils.FilesystemRoot) - } - return utils.FilesystemRoot -} diff --git a/pkg/host/interface.go b/pkg/host/interface.go new file mode 100644 index 000000000..edccfd5c8 --- /dev/null +++ b/pkg/host/interface.go @@ -0,0 +1,55 @@ +package host + +import ( + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +const ( + hostPathFromDaemon = consts.Host + redhatReleaseFile = "/etc/redhat-release" + rhelRDMAConditionFile = "/usr/libexec/rdma-init-kernel" + rhelRDMAServiceName = "rdma" + rhelPackageManager = "yum" + + ubuntuRDMAConditionFile = "/usr/sbin/rdma-ndd" + ubuntuRDMAServiceName = "rdma-ndd" + ubuntuPackageManager = "apt-get" + + genericOSReleaseFile = "/etc/os-release" +) + +// Contains all the host manipulation functions +type HostManagerInterface interface { + KernelInterface + NetworkInterface + ServiceInterface + UdevInterface + SriovInterface +} + +type hostManager struct { + utils.CmdInterface + KernelInterface + NetworkInterface + ServiceInterface + UdevInterface + SriovInterface +} + +func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { + k := newKernelInterface(utilsInterface) + n := newNetworkInterface(utilsInterface) + sv := newServiceInterface(utilsInterface) + u := newUdevInterface(utilsInterface) + sr := newSriovInterface(utilsInterface, k, n, u) + + return &hostManager{ + utilsInterface, + k, + n, + sv, + u, + sr, + } +} diff --git a/pkg/host/kernel.go b/pkg/host/kernel.go new file mode 100644 index 000000000..ac7277eef --- /dev/null +++ b/pkg/host/kernel.go @@ -0,0 +1,627 @@ +package host + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +type KernelInterface interface { + // TryEnableTun load the tun kernel module + TryEnableTun() + // TryEnableVhostNet load the vhost-net kernel module + TryEnableVhostNet() + // TryEnableRdma tries to enable RDMA on the machine base on the operating system + // if the package doesn't exist it will also will try to install it + // supported operating systems are RHEL RHCOS and ubuntu + TryEnableRdma() (bool, error) + // TriggerUdevEvent triggers a udev event + TriggerUdevEvent() error + // GetCurrentKernelArgs reads the /proc/cmdline to check the current kernel arguments + GetCurrentKernelArgs() (string, error) + // IsKernelArgsSet check is the requested kernel arguments are set + IsKernelArgsSet(string, string) bool + // Unbind unbinds a virtual function from is current driver + Unbind(string) error + // BindDpdkDriver binds the virtual function to a DPDK driver + BindDpdkDriver(string, string) error + // BindDefaultDriver binds the virtual function to is default driver + BindDefaultDriver(string) error + // HasDriver returns try if the virtual function is bind to a driver + HasDriver(string) (bool, string) + // RebindVfToDefaultDriver rebinds the virtual function to is default driver + RebindVfToDefaultDriver(string) error + // UnbindDriverIfNeeded unbinds the virtual function from a driver if needed + UnbindDriverIfNeeded(string, bool) error + // LoadKernelModule loads a kernel module to the host + LoadKernelModule(name string, args ...string) error + // IsKernelModuleLoaded returns try if the requested kernel module is loaded + IsKernelModuleLoaded(string) (bool, error) + // ReloadDriver reloads a requested driver + ReloadDriver(string) error + // IsKernelLockdownMode returns true if the kernel is in lockdown mode + IsKernelLockdownMode() bool + // IsRHELSystem returns try if the system is a RHEL base + IsRHELSystem() (bool, error) + // IsUbuntuSystem returns try if the system is an ubuntu base + IsUbuntuSystem() (bool, error) + // IsCoreOS returns true if the system is a CoreOS or RHCOS base + IsCoreOS() (bool, error) + // RdmaIsLoaded returns try if RDMA kernel modules are loaded + RdmaIsLoaded() (bool, error) + // EnableRDMA enable RDMA on the system + EnableRDMA(string, string, string) (bool, error) + // InstallRDMA install RDMA packages on the system + InstallRDMA(string) error + // EnableRDMAOnRHELMachine enable RDMA on a RHEL base system + EnableRDMAOnRHELMachine() (bool, error) + // GetOSPrettyName returns OS name + GetOSPrettyName() (string, error) +} + +type kernel struct { + utilsHelper utils.CmdInterface +} + +func newKernelInterface(utilsHelper utils.CmdInterface) KernelInterface { + return &kernel{utilsHelper: utilsHelper} +} + +func (k *kernel) LoadKernelModule(name string, args ...string) error { + log.Log.Info("LoadKernelModule(): try to load kernel module", "name", name, "args", args) + chrootDefinition := utils.GetChrootExtension() + cmdArgs := strings.Join(args, " ") + + // check if the driver is already loaded in to the system + isLoaded, err := k.IsKernelModuleLoaded(name) + if err != nil { + log.Log.Error(err, "LoadKernelModule(): failed to check if kernel module is already loaded", "name", name) + } + if isLoaded { + log.Log.Info("LoadKernelModule(): kernel module already loaded", "name", name) + return nil + } + + _, _, err = k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s modprobe %s %s", chrootDefinition, name, cmdArgs)) + if err != nil { + log.Log.Error(err, "LoadKernelModule(): failed to load kernel module with arguments", "name", name, "args", args) + return err + } + return nil +} + +func (k *kernel) IsKernelModuleLoaded(kernelModuleName string) (bool, error) { + log.Log.Info("IsKernelModuleLoaded(): check if kernel module is loaded", "name", kernelModuleName) + chrootDefinition := utils.GetChrootExtension() + + stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s lsmod | grep \"^%s\"", chrootDefinition, kernelModuleName)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", + "name", kernelModuleName, "stderr", stderr) + return false, err + } + log.Log.V(2).Info("IsKernelModuleLoaded():", "stdout", stdout) + if len(stderr) != 0 { + log.Log.Error(err, "IsKernelModuleLoaded(): failed to check if kernel module is loaded", "name", kernelModuleName, "stderr", stderr) + return false, fmt.Errorf(stderr) + } + + if len(stdout) != 0 { + log.Log.Info("IsKernelModuleLoaded(): kernel module already loaded", "name", kernelModuleName) + return true, nil + } + + return false, nil +} + +func (k *kernel) TryEnableTun() { + if err := k.LoadKernelModule("tun"); err != nil { + log.Log.Error(err, "tryEnableTun(): TUN kernel module not loaded") + } +} + +func (k *kernel) TryEnableVhostNet() { + if err := k.LoadKernelModule("vhost_net"); err != nil { + log.Log.Error(err, "tryEnableVhostNet(): VHOST_NET kernel module not loaded") + } +} + +// GetCurrentKernelArgs This retrieves the kernel cmd line arguments +func (k *kernel) GetCurrentKernelArgs() (string, error) { + path := consts.ProcKernelCmdLine + if !vars.UsingSystemdMode { + path = filepath.Join(consts.Host, path) + } + + path = filepath.Join(vars.FilesystemRoot, path) + cmdLine, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("GetCurrentKernelArgs(): Error reading %s: %v", path, err) + } + return string(cmdLine), nil +} + +// IsKernelArgsSet This checks if the kernel cmd line is set properly. Please note that the same key could be repeated +// several times in the kernel cmd line. We can only ensure that the kernel cmd line has the key/val kernel arg that we set. +func (k *kernel) IsKernelArgsSet(cmdLine string, karg string) bool { + elements := strings.Fields(cmdLine) + for _, element := range elements { + if element == karg { + return true + } + } + return false +} + +// Unbind unbind driver for one device +func (k *kernel) Unbind(pciAddr string) error { + log.Log.V(2).Info("Unbind(): unbind device driver for device", "device", pciAddr) + yes, driver := k.HasDriver(pciAddr) + if !yes { + return nil + } + + filePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDrivers, driver, "unbind") + err := os.WriteFile(filePath, []byte(pciAddr), os.ModeAppend) + if err != nil { + log.Log.Error(err, "Unbind(): fail to unbind driver for device", "device", pciAddr) + return err + } + return nil +} + +// BindDpdkDriver bind dpdk driver for one device +// Bind the device given by "pciAddr" to the driver "driver" +func (k *kernel) BindDpdkDriver(pciAddr, driver string) error { + log.Log.V(2).Info("BindDpdkDriver(): bind device to driver", + "device", pciAddr, "driver", driver) + + if yes, d := k.HasDriver(pciAddr); yes { + if driver == d { + log.Log.V(2).Info("BindDpdkDriver(): device already bound to driver", + "device", pciAddr, "driver", driver) + return nil + } + + if err := k.Unbind(pciAddr); err != nil { + return err + } + } + + driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "driver_override") + err := os.WriteFile(driverOverridePath, []byte(driver), os.ModeAppend) + if err != nil { + log.Log.Error(err, "BindDpdkDriver(): fail to write driver_override for device", + "device", pciAddr, "driver", driver) + return err + } + bindPath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDrivers, driver, "bind") + err = os.WriteFile(bindPath, []byte(pciAddr), os.ModeAppend) + if err != nil { + log.Log.Error(err, "BindDpdkDriver(): fail to bind driver for device", + "driver", driver, "device", pciAddr) + _, err := os.Readlink(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "iommu_group")) + if err != nil { + log.Log.Error(err, "Could not read IOMMU group for device", "device", pciAddr) + return fmt.Errorf( + "cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, err) + } + return err + } + err = os.WriteFile(driverOverridePath, []byte(""), os.ModeAppend) + if err != nil { + log.Log.Error(err, "BindDpdkDriver(): failed to clear driver_override for device", "device", pciAddr) + return err + } + + return nil +} + +// BindDefaultDriver bind driver for one device +// Bind the device given by "pciAddr" to the default driver +func (k *kernel) BindDefaultDriver(pciAddr string) error { + log.Log.V(2).Info("BindDefaultDriver(): bind device to default driver", "device", pciAddr) + + if yes, d := k.HasDriver(pciAddr); yes { + if !sriovnetworkv1.StringInArray(d, vars.DpdkDrivers) { + log.Log.V(2).Info("BindDefaultDriver(): device already bound to default driver", + "device", pciAddr, "driver", d) + return nil + } + if err := k.Unbind(pciAddr); err != nil { + return err + } + } + + driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "driver_override") + err := os.WriteFile(driverOverridePath, []byte("\x00"), os.ModeAppend) + if err != nil { + log.Log.Error(err, "BindDefaultDriver(): failed to write driver_override for device", "device", pciAddr) + return err + } + + pciDriversProbe := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDriversProbe) + err = os.WriteFile(pciDriversProbe, []byte(pciAddr), os.ModeAppend) + if err != nil { + log.Log.Error(err, "BindDefaultDriver(): failed to bind driver for device", "device", pciAddr) + return err + } + + return nil +} + +// Workaround function to handle a case where the vf default driver is stuck and not able to create the vf kernel interface. +// This function unbind the VF from the default driver and try to bind it again +// bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2045087 +func (k *kernel) RebindVfToDefaultDriver(vfAddr string) error { + log.Log.Info("RebindVfToDefaultDriver()", "vf", vfAddr) + if err := k.Unbind(vfAddr); err != nil { + return err + } + if err := k.BindDefaultDriver(vfAddr); err != nil { + log.Log.Error(err, "RebindVfToDefaultDriver(): fail to bind default driver", "device", vfAddr) + return err + } + + log.Log.Info("RebindVfToDefaultDriver(): workaround implemented", "vf", vfAddr) + return nil +} + +func (k *kernel) UnbindDriverIfNeeded(vfAddr string, isRdma bool) error { + if isRdma { + log.Log.Info("UnbindDriverIfNeeded(): unbinding driver", "device", vfAddr) + if err := k.Unbind(vfAddr); err != nil { + return err + } + log.Log.Info("UnbindDriverIfNeeded(): unbounded driver", "device", vfAddr) + } + return nil +} + +func (k *kernel) HasDriver(pciAddr string) (bool, string) { + driver, err := dputils.GetDriverName(pciAddr) + if err != nil { + log.Log.V(2).Info("HasDriver(): device driver is empty for device", "device", pciAddr) + return false, "" + } + log.Log.V(2).Info("HasDriver(): device driver for device", "device", pciAddr, "driver", driver) + return true, driver +} + +func (k *kernel) TryEnableRdma() (bool, error) { + log.Log.V(2).Info("tryEnableRdma()") + chrootDefinition := utils.GetChrootExtension() + + // check if the driver is already loaded in to the system + _, stderr, mlx4Err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("grep --quiet 'mlx4_en' <(%s lsmod)", chrootDefinition)) + if mlx4Err != nil && len(stderr) != 0 { + log.Log.Error(mlx4Err, "tryEnableRdma(): failed to check for kernel module 'mlx4_en'", "stderr", stderr) + return false, fmt.Errorf(stderr) + } + + _, stderr, mlx5Err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("grep --quiet 'mlx5_core' <(%s lsmod)", chrootDefinition)) + if mlx5Err != nil && len(stderr) != 0 { + log.Log.Error(mlx5Err, "tryEnableRdma(): failed to check for kernel module 'mlx5_core'", "stderr", stderr) + return false, fmt.Errorf(stderr) + } + + if mlx4Err != nil && mlx5Err != nil { + log.Log.Error(nil, "tryEnableRdma(): no RDMA capable devices") + return false, nil + } + + isRhelSystem, err := k.IsRHELSystem() + if err != nil { + log.Log.Error(err, "tryEnableRdma(): failed to check if the machine is base on RHEL") + return false, err + } + + // RHEL check + if isRhelSystem { + return k.EnableRDMAOnRHELMachine() + } + + isUbuntuSystem, err := k.IsUbuntuSystem() + if err != nil { + log.Log.Error(err, "tryEnableRdma(): failed to check if the machine is base on Ubuntu") + return false, err + } + + if isUbuntuSystem { + return k.EnableRDMAOnUbuntuMachine() + } + + osName, err := k.GetOSPrettyName() + if err != nil { + log.Log.Error(err, "tryEnableRdma(): failed to check OS name") + return false, err + } + + log.Log.Error(nil, "tryEnableRdma(): Unsupported OS", "name", osName) + return false, fmt.Errorf("unable to load RDMA unsupported OS: %s", osName) +} + +func (k *kernel) EnableRDMAOnRHELMachine() (bool, error) { + log.Log.Info("EnableRDMAOnRHELMachine()") + isCoreOsSystem, err := k.IsCoreOS() + if err != nil { + log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if the machine runs CoreOS") + return false, err + } + + // CoreOS check + if isCoreOsSystem { + isRDMALoaded, err := k.RdmaIsLoaded() + if err != nil { + log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if RDMA kernel modules are loaded") + return false, err + } + + return isRDMALoaded, nil + } + + // RHEL + log.Log.Info("EnableRDMAOnRHELMachine(): enabling RDMA on RHEL machine") + isRDMAEnable, err := k.EnableRDMA(rhelRDMAConditionFile, rhelRDMAServiceName, rhelPackageManager) + if err != nil { + log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to enable RDMA on RHEL machine") + return false, err + } + + // check if we need to install rdma-core package + if isRDMAEnable { + isRDMALoaded, err := k.RdmaIsLoaded() + if err != nil { + log.Log.Error(err, "EnableRDMAOnRHELMachine(): failed to check if RDMA kernel modules are loaded") + return false, err + } + + // if ib kernel module is not loaded trigger a loading + if isRDMALoaded { + err = k.TriggerUdevEvent() + if err != nil { + log.Log.Error(err, "EnableRDMAOnRHELMachine() failed to trigger udev event") + return false, err + } + } + } + + return true, nil +} + +func (k *kernel) EnableRDMAOnUbuntuMachine() (bool, error) { + log.Log.Info("EnableRDMAOnUbuntuMachine(): enabling RDMA on RHEL machine") + isRDMAEnable, err := k.EnableRDMA(ubuntuRDMAConditionFile, ubuntuRDMAServiceName, ubuntuPackageManager) + if err != nil { + log.Log.Error(err, "EnableRDMAOnUbuntuMachine(): failed to enable RDMA on Ubuntu machine") + return false, err + } + + // check if we need to install rdma-core package + if isRDMAEnable { + isRDMALoaded, err := k.RdmaIsLoaded() + if err != nil { + log.Log.Error(err, "EnableRDMAOnUbuntuMachine(): failed to check if RDMA kernel modules are loaded") + return false, err + } + + // if ib kernel module is not loaded trigger a loading + if isRDMALoaded { + err = k.TriggerUdevEvent() + if err != nil { + log.Log.Error(err, "EnableRDMAOnUbuntuMachine() failed to trigger udev event") + return false, err + } + } + } + + return true, nil +} + +func (k *kernel) IsRHELSystem() (bool, error) { + log.Log.Info("IsRHELSystem(): checking for RHEL machine") + path := redhatReleaseFile + if !vars.UsingSystemdMode { + path = filepath.Join(hostPathFromDaemon, path) + } + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + log.Log.V(2).Info("IsRHELSystem() not a RHEL machine") + return false, nil + } + + log.Log.Error(err, "IsRHELSystem() failed to check for os release file", "path", path) + return false, err + } + + return true, nil +} + +func (k *kernel) IsCoreOS() (bool, error) { + log.Log.Info("IsCoreOS(): checking for CoreOS machine") + path := redhatReleaseFile + if !vars.UsingSystemdMode { + path = filepath.Join(hostPathFromDaemon, path) + } + + data, err := os.ReadFile(path) + if err != nil { + log.Log.Error(err, "IsCoreOS(): failed to read RHEL release file on path", "path", path) + return false, err + } + + if strings.Contains(string(data), "CoreOS") { + return true, nil + } + + return false, nil +} + +func (k *kernel) IsUbuntuSystem() (bool, error) { + log.Log.Info("IsUbuntuSystem(): checking for Ubuntu machine") + path := genericOSReleaseFile + if !vars.UsingSystemdMode { + path = filepath.Join(hostPathFromDaemon, path) + } + + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + log.Log.Error(nil, "IsUbuntuSystem() os-release on path doesn't exist", "path", path) + return false, err + } + + log.Log.Error(err, "IsUbuntuSystem() failed to check for os release file", "path", path) + return false, err + } + + stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("grep -i --quiet 'ubuntu' %s", path)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "IsUbuntuSystem(): failed to check for ubuntu operating system name in os-releasae file", "stderr", stderr) + return false, fmt.Errorf(stderr) + } + + if len(stdout) > 0 { + return true, nil + } + + return false, nil +} + +func (k *kernel) RdmaIsLoaded() (bool, error) { + log.Log.V(2).Info("RdmaIsLoaded()") + chrootDefinition := utils.GetChrootExtension() + + // check if the driver is already loaded in to the system + _, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("grep --quiet '\\(^ib\\|^rdma\\)' <(%s lsmod)", chrootDefinition)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "RdmaIsLoaded(): fail to check if ib and rdma kernel modules are loaded", "stderr", stderr) + return false, fmt.Errorf(stderr) + } + + if err != nil { + return false, nil + } + + return true, nil +} + +func (k *kernel) EnableRDMA(conditionFilePath, serviceName, packageManager string) (bool, error) { + path := conditionFilePath + if !vars.UsingSystemdMode { + path = filepath.Join(hostPathFromDaemon, path) + } + log.Log.Info("EnableRDMA(): checking for service file", "path", path) + + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + log.Log.V(2).Info("EnableRDMA(): RDMA server doesn't exist") + err = k.InstallRDMA(packageManager) + if err != nil { + log.Log.Error(err, "EnableRDMA() failed to install RDMA package") + return false, err + } + + err = k.TriggerUdevEvent() + if err != nil { + log.Log.Error(err, "EnableRDMA() failed to trigger udev event") + return false, err + } + + return false, nil + } + + log.Log.Error(err, "EnableRDMA() failed to check for os release file", "path", path) + return false, err + } + + log.Log.Info("EnableRDMA(): service installed", "name", serviceName) + return true, nil +} + +func (k *kernel) InstallRDMA(packageManager string) error { + log.Log.Info("InstallRDMA(): installing RDMA") + chrootDefinition := utils.GetChrootExtension() + + stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s %s install -y rdma-core", chrootDefinition, packageManager)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "InstallRDMA(): failed to install RDMA package", "stdout", stdout, "stderr", stderr) + return err + } + + return nil +} + +func (k *kernel) TriggerUdevEvent() error { + log.Log.Info("TriggerUdevEvent(): installing RDMA") + + err := k.ReloadDriver("mlx4_en") + if err != nil { + return err + } + + err = k.ReloadDriver("mlx5_core") + if err != nil { + return err + } + + return nil +} + +func (k *kernel) ReloadDriver(driverName string) error { + log.Log.Info("ReloadDriver(): reload driver", "name", driverName) + chrootDefinition := utils.GetChrootExtension() + + _, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("%s modprobe -r %s && %s modprobe %s", chrootDefinition, driverName, chrootDefinition, driverName)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "InstallRDMA(): failed to reload kernel module", + "name", driverName, "stderr", stderr) + return err + } + + return nil +} + +func (k *kernel) GetOSPrettyName() (string, error) { + path := genericOSReleaseFile + if !vars.UsingSystemdMode { + path = filepath.Join(hostPathFromDaemon, path) + } + + log.Log.Info("GetOSPrettyName(): getting os name from os-release file") + + stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", fmt.Sprintf("cat %s | grep PRETTY_NAME | cut -c 13-", path)) + if err != nil && len(stderr) != 0 { + log.Log.Error(err, "IsUbuntuSystem(): failed to check for ubuntu operating system name in os-releasae file", "stderr", stderr) + return "", fmt.Errorf(stderr) + } + + if len(stdout) > 0 { + return stdout, nil + } + + return "", fmt.Errorf("failed to find pretty operating system name") +} + +// IsKernelLockdownMode returns true when kernel lockdown mode is enabled +// TODO: change this to return error +func (k *kernel) IsKernelLockdownMode() bool { + path := utils.GetHostExtension() + path = filepath.Join(path, "/sys/kernel/security/lockdown") + + stdout, stderr, err := k.utilsHelper.RunCommand("/bin/sh", "-c", "cat", path) + log.Log.V(2).Info("IsKernelLockdownMode()", "output", stdout, "error", err) + if err != nil { + log.Log.Error(err, "IsKernelLockdownMode(): failed to check for lockdown file", "stderr", stderr) + return false + } + return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]") +} diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go deleted file mode 100644 index e09bfd3c8..000000000 --- a/pkg/host/mock/mock_host.go +++ /dev/null @@ -1,254 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: host.go - -// Package mock_host is a generated GoMock package. -package mock_host - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockHostManagerInterface is a mock of HostManagerInterface interface. -type MockHostManagerInterface struct { - ctrl *gomock.Controller - recorder *MockHostManagerInterfaceMockRecorder -} - -// MockHostManagerInterfaceMockRecorder is the mock recorder for MockHostManagerInterface. -type MockHostManagerInterfaceMockRecorder struct { - mock *MockHostManagerInterface -} - -// NewMockHostManagerInterface creates a new mock instance. -func NewMockHostManagerInterface(ctrl *gomock.Controller) *MockHostManagerInterface { - mock := &MockHostManagerInterface{ctrl: ctrl} - mock.recorder = &MockHostManagerInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockHostManagerInterface) EXPECT() *MockHostManagerInterfaceMockRecorder { - return m.recorder -} - -// EnableRDMA mocks base method. -func (m *MockHostManagerInterface) EnableRDMA(arg0, arg1, arg2 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnableRDMA", arg0, arg1, arg2) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EnableRDMA indicates an expected call of EnableRDMA. -func (mr *MockHostManagerInterfaceMockRecorder) EnableRDMA(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableRDMA", reflect.TypeOf((*MockHostManagerInterface)(nil).EnableRDMA), arg0, arg1, arg2) -} - -// EnableRDMAOnRHELMachine mocks base method. -func (m *MockHostManagerInterface) EnableRDMAOnRHELMachine() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnableRDMAOnRHELMachine") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EnableRDMAOnRHELMachine indicates an expected call of EnableRDMAOnRHELMachine. -func (mr *MockHostManagerInterfaceMockRecorder) EnableRDMAOnRHELMachine() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableRDMAOnRHELMachine", reflect.TypeOf((*MockHostManagerInterface)(nil).EnableRDMAOnRHELMachine)) -} - -// GetOSPrettyName mocks base method. -func (m *MockHostManagerInterface) GetOSPrettyName() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetOSPrettyName") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetOSPrettyName indicates an expected call of GetOSPrettyName. -func (mr *MockHostManagerInterfaceMockRecorder) GetOSPrettyName() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOSPrettyName", reflect.TypeOf((*MockHostManagerInterface)(nil).GetOSPrettyName)) -} - -// InstallRDMA mocks base method. -func (m *MockHostManagerInterface) InstallRDMA(arg0 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallRDMA", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// InstallRDMA indicates an expected call of InstallRDMA. -func (mr *MockHostManagerInterfaceMockRecorder) InstallRDMA(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallRDMA", reflect.TypeOf((*MockHostManagerInterface)(nil).InstallRDMA), arg0) -} - -// IsCoreOS mocks base method. -func (m *MockHostManagerInterface) IsCoreOS() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsCoreOS") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsCoreOS indicates an expected call of IsCoreOS. -func (mr *MockHostManagerInterfaceMockRecorder) IsCoreOS() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCoreOS", reflect.TypeOf((*MockHostManagerInterface)(nil).IsCoreOS)) -} - -// IsKernelModuleLoaded mocks base method. -func (m *MockHostManagerInterface) IsKernelModuleLoaded(arg0 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsKernelModuleLoaded", arg0) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsKernelModuleLoaded indicates an expected call of IsKernelModuleLoaded. -func (mr *MockHostManagerInterfaceMockRecorder) IsKernelModuleLoaded(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelModuleLoaded", reflect.TypeOf((*MockHostManagerInterface)(nil).IsKernelModuleLoaded), arg0) -} - -// IsRHELSystem mocks base method. -func (m *MockHostManagerInterface) IsRHELSystem() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsRHELSystem") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsRHELSystem indicates an expected call of IsRHELSystem. -func (mr *MockHostManagerInterfaceMockRecorder) IsRHELSystem() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRHELSystem", reflect.TypeOf((*MockHostManagerInterface)(nil).IsRHELSystem)) -} - -// IsUbuntuSystem mocks base method. -func (m *MockHostManagerInterface) IsUbuntuSystem() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsUbuntuSystem") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsUbuntuSystem indicates an expected call of IsUbuntuSystem. -func (mr *MockHostManagerInterfaceMockRecorder) IsUbuntuSystem() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUbuntuSystem", reflect.TypeOf((*MockHostManagerInterface)(nil).IsUbuntuSystem)) -} - -// LoadKernelModule mocks base method. -func (m *MockHostManagerInterface) LoadKernelModule(name string, args ...string) error { - m.ctrl.T.Helper() - varargs := []interface{}{name} - for _, a := range args { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "LoadKernelModule", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// LoadKernelModule indicates an expected call of LoadKernelModule. -func (mr *MockHostManagerInterfaceMockRecorder) LoadKernelModule(name interface{}, args ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{name}, args...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadKernelModule", reflect.TypeOf((*MockHostManagerInterface)(nil).LoadKernelModule), varargs...) -} - -// RdmaIsLoaded mocks base method. -func (m *MockHostManagerInterface) RdmaIsLoaded() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RdmaIsLoaded") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// RdmaIsLoaded indicates an expected call of RdmaIsLoaded. -func (mr *MockHostManagerInterfaceMockRecorder) RdmaIsLoaded() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaIsLoaded", reflect.TypeOf((*MockHostManagerInterface)(nil).RdmaIsLoaded)) -} - -// ReloadDriver mocks base method. -func (m *MockHostManagerInterface) ReloadDriver(arg0 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReloadDriver", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ReloadDriver indicates an expected call of ReloadDriver. -func (mr *MockHostManagerInterfaceMockRecorder) ReloadDriver(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReloadDriver", reflect.TypeOf((*MockHostManagerInterface)(nil).ReloadDriver), arg0) -} - -// TriggerUdevEvent mocks base method. -func (m *MockHostManagerInterface) TriggerUdevEvent() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TriggerUdevEvent") - ret0, _ := ret[0].(error) - return ret0 -} - -// TriggerUdevEvent indicates an expected call of TriggerUdevEvent. -func (mr *MockHostManagerInterfaceMockRecorder) TriggerUdevEvent() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerUdevEvent", reflect.TypeOf((*MockHostManagerInterface)(nil).TriggerUdevEvent)) -} - -// TryEnableRdma mocks base method. -func (m *MockHostManagerInterface) TryEnableRdma() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TryEnableRdma") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// TryEnableRdma indicates an expected call of TryEnableRdma. -func (mr *MockHostManagerInterfaceMockRecorder) TryEnableRdma() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableRdma", reflect.TypeOf((*MockHostManagerInterface)(nil).TryEnableRdma)) -} - -// TryEnableTun mocks base method. -func (m *MockHostManagerInterface) TryEnableTun() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "TryEnableTun") -} - -// TryEnableTun indicates an expected call of TryEnableTun. -func (mr *MockHostManagerInterfaceMockRecorder) TryEnableTun() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableTun", reflect.TypeOf((*MockHostManagerInterface)(nil).TryEnableTun)) -} - -// TryEnableVhostNet mocks base method. -func (m *MockHostManagerInterface) TryEnableVhostNet() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "TryEnableVhostNet") -} - -// TryEnableVhostNet indicates an expected call of TryEnableVhostNet. -func (mr *MockHostManagerInterfaceMockRecorder) TryEnableVhostNet() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryEnableVhostNet", reflect.TypeOf((*MockHostManagerInterface)(nil).TryEnableVhostNet)) -} diff --git a/pkg/host/mock/mock_store.go b/pkg/host/mock/mock_store.go new file mode 100644 index 000000000..54eae5850 --- /dev/null +++ b/pkg/host/mock/mock_store.go @@ -0,0 +1,108 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: store.go + +// Package mock_host is a generated GoMock package. +package mock_host + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +// MockStoreManagerInterface is a mock of StoreManagerInterface interface. +type MockStoreManagerInterface struct { + ctrl *gomock.Controller + recorder *MockStoreManagerInterfaceMockRecorder +} + +// MockStoreManagerInterfaceMockRecorder is the mock recorder for MockStoreManagerInterface. +type MockStoreManagerInterfaceMockRecorder struct { + mock *MockStoreManagerInterface +} + +// NewMockStoreManagerInterface creates a new mock instance. +func NewMockStoreManagerInterface(ctrl *gomock.Controller) *MockStoreManagerInterface { + mock := &MockStoreManagerInterface{ctrl: ctrl} + mock.recorder = &MockStoreManagerInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStoreManagerInterface) EXPECT() *MockStoreManagerInterfaceMockRecorder { + return m.recorder +} + +// ClearPCIAddressFolder mocks base method. +func (m *MockStoreManagerInterface) ClearPCIAddressFolder() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearPCIAddressFolder") + ret0, _ := ret[0].(error) + return ret0 +} + +// ClearPCIAddressFolder indicates an expected call of ClearPCIAddressFolder. +func (mr *MockStoreManagerInterfaceMockRecorder) ClearPCIAddressFolder() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearPCIAddressFolder", reflect.TypeOf((*MockStoreManagerInterface)(nil).ClearPCIAddressFolder)) +} + +// GetCheckPointNodeState mocks base method. +func (m *MockStoreManagerInterface) GetCheckPointNodeState() (*v1.SriovNetworkNodeState, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCheckPointNodeState") + ret0, _ := ret[0].(*v1.SriovNetworkNodeState) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCheckPointNodeState indicates an expected call of GetCheckPointNodeState. +func (mr *MockStoreManagerInterfaceMockRecorder) GetCheckPointNodeState() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCheckPointNodeState", reflect.TypeOf((*MockStoreManagerInterface)(nil).GetCheckPointNodeState)) +} + +// LoadPfsStatus mocks base method. +func (m *MockStoreManagerInterface) LoadPfsStatus(pciAddress string) (*v1.Interface, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadPfsStatus", pciAddress) + ret0, _ := ret[0].(*v1.Interface) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// LoadPfsStatus indicates an expected call of LoadPfsStatus. +func (mr *MockStoreManagerInterfaceMockRecorder) LoadPfsStatus(pciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPfsStatus", reflect.TypeOf((*MockStoreManagerInterface)(nil).LoadPfsStatus), pciAddress) +} + +// SaveLastPfAppliedStatus mocks base method. +func (m *MockStoreManagerInterface) SaveLastPfAppliedStatus(PfInfo *v1.Interface) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveLastPfAppliedStatus", PfInfo) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveLastPfAppliedStatus indicates an expected call of SaveLastPfAppliedStatus. +func (mr *MockStoreManagerInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveLastPfAppliedStatus", reflect.TypeOf((*MockStoreManagerInterface)(nil).SaveLastPfAppliedStatus), PfInfo) +} + +// WriteCheckpointFile mocks base method. +func (m *MockStoreManagerInterface) WriteCheckpointFile(arg0 *v1.SriovNetworkNodeState) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteCheckpointFile", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteCheckpointFile indicates an expected call of WriteCheckpointFile. +func (mr *MockStoreManagerInterfaceMockRecorder) WriteCheckpointFile(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteCheckpointFile", reflect.TypeOf((*MockStoreManagerInterface)(nil).WriteCheckpointFile), arg0) +} diff --git a/pkg/host/network.go b/pkg/host/network.go new file mode 100644 index 000000000..14885f6b9 --- /dev/null +++ b/pkg/host/network.go @@ -0,0 +1,215 @@ +package host + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/cenkalti/backoff" + "sigs.k8s.io/controller-runtime/pkg/log" + + dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +type NetworkInterface interface { + // TryToGetVirtualInterfaceName tries to find the virtio interface name base on pci address + // used for virtual environment where we pass SR-IOV virtual function into the system + // supported platform openstack + TryToGetVirtualInterfaceName(string) string + // TryGetInterfaceName tries to find the SR-IOV virtual interface name base on pci address + TryGetInterfaceName(string) string + // GetPhysSwitchID returns the physical switch ID for a specific pci address + GetPhysSwitchID(string) (string, error) + // GetPhysPortName returns the physical port name for a specific pci address + GetPhysPortName(string) (string, error) + // IsSwitchdev returns true of the pci address is on switchdev mode + IsSwitchdev(string) bool + // GetNetdevMTU returns the interface MTU for devices attached to kernel drivers + GetNetdevMTU(string) int + // SetNetdevMTU sets the MTU for a request interface + SetNetdevMTU(string, int) error + // GetNetDevMac returns the network interface mac address + GetNetDevMac(string) string + // GetNetDevLinkSpeed returns the network interface link speed + GetNetDevLinkSpeed(string) string +} + +type network struct { + utilsHelper utils.CmdInterface +} + +func newNetworkInterface(utilsHelper utils.CmdInterface) NetworkInterface { + return &network{utilsHelper: utilsHelper} +} + +// TryToGetVirtualInterfaceName get the interface name of a virtio interface +func (n *network) TryToGetVirtualInterfaceName(pciAddr string) string { + log.Log.Info("TryToGetVirtualInterfaceName() get interface name for device", "device", pciAddr) + + // To support different driver that is not virtio-pci like mlx + name := n.TryGetInterfaceName(pciAddr) + if name != "" { + return name + } + + netDir, err := filepath.Glob(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "virtio*", "net")) + if err != nil || len(netDir) < 1 { + return "" + } + + fInfos, err := os.ReadDir(netDir[0]) + if err != nil { + log.Log.Error(err, "TryToGetVirtualInterfaceName(): failed to read net directory", "dir", netDir[0]) + return "" + } + + names := make([]string, 0) + for _, f := range fInfos { + names = append(names, f.Name()) + } + + if len(names) < 1 { + return "" + } + + return names[0] +} + +func (n *network) TryGetInterfaceName(pciAddr string) string { + names, err := dputils.GetNetNames(pciAddr) + if err != nil || len(names) < 1 { + return "" + } + netDevName := names[0] + + // Switchdev PF and their VFs representors are existing under the same PCI address since kernel 5.8 + // if device is switchdev then return PF name + for _, name := range names { + if !n.IsSwitchdev(name) { + continue + } + // Try to get the phys port name, if not exists then fallback to check without it + // phys_port_name should be in formant p e.g p0,p1,p2 ...etc. + if physPortName, err := n.GetPhysPortName(name); err == nil { + if !vars.PfPhysPortNameRe.MatchString(physPortName) { + continue + } + } + return name + } + + log.Log.V(2).Info("tryGetInterfaceName()", "name", netDevName) + return netDevName +} + +func (n *network) GetPhysSwitchID(name string) (string, error) { + swIDFile := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, name, "phys_switch_id") + physSwitchID, err := os.ReadFile(swIDFile) + if err != nil { + return "", err + } + if physSwitchID != nil { + return strings.TrimSpace(string(physSwitchID)), nil + } + return "", nil +} + +func (n *network) GetPhysPortName(name string) (string, error) { + devicePortNameFile := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, name, "phys_port_name") + physPortName, err := os.ReadFile(devicePortNameFile) + if err != nil { + return "", err + } + if physPortName != nil { + return strings.TrimSpace(string(physPortName)), nil + } + return "", nil +} + +func (n *network) IsSwitchdev(name string) bool { + switchID, err := n.GetPhysSwitchID(name) + if err != nil || switchID == "" { + return false + } + + return true +} + +func (n *network) GetNetdevMTU(pciAddr string) int { + log.Log.V(2).Info("GetNetdevMTU(): get MTU", "device", pciAddr) + ifaceName := n.TryGetInterfaceName(pciAddr) + if ifaceName == "" { + return 0 + } + mtuFile := "net/" + ifaceName + "/mtu" + mtuFilePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, mtuFile) + data, err := os.ReadFile(mtuFilePath) + if err != nil { + log.Log.Error(err, "GetNetdevMTU(): fail to read mtu file", "path", mtuFilePath) + return 0 + } + mtu, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + log.Log.Error(err, "GetNetdevMTU(): fail to convert mtu to int", "raw-mtu", strings.TrimSpace(string(data))) + return 0 + } + return mtu +} + +func (n *network) SetNetdevMTU(pciAddr string, mtu int) error { + log.Log.V(2).Info("SetNetdevMTU(): set MTU", "device", pciAddr, "mtu", mtu) + if mtu <= 0 { + log.Log.V(2).Info("SetNetdevMTU(): refusing to set MTU", "mtu", mtu) + return nil + } + b := backoff.NewConstantBackOff(1 * time.Second) + err := backoff.Retry(func() error { + ifaceName, err := dputils.GetNetNames(pciAddr) + if err != nil { + log.Log.Error(err, "SetNetdevMTU(): fail to get interface name", "device", pciAddr) + return err + } + if len(ifaceName) < 1 { + return fmt.Errorf("SetNetdevMTU(): interface name is empty") + } + mtuFile := "net/" + ifaceName[0] + "/mtu" + mtuFilePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, mtuFile) + return os.WriteFile(mtuFilePath, []byte(strconv.Itoa(mtu)), os.ModeAppend) + }, backoff.WithMaxRetries(b, 10)) + if err != nil { + log.Log.Error(err, "SetNetdevMTU(): fail to write mtu file after retrying") + return err + } + return nil +} + +func (n *network) GetNetDevMac(ifaceName string) string { + log.Log.V(2).Info("GetNetDevMac(): get Mac", "device", ifaceName) + macFilePath := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, ifaceName, "address") + data, err := os.ReadFile(macFilePath) + if err != nil { + log.Log.Error(err, "GetNetDevMac(): fail to read Mac file", "path", macFilePath) + return "" + } + + return strings.TrimSpace(string(data)) +} + +func (n *network) GetNetDevLinkSpeed(ifaceName string) string { + log.Log.V(2).Info("GetNetDevLinkSpeed(): get LinkSpeed", "device", ifaceName) + speedFilePath := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, ifaceName, "speed") + data, err := os.ReadFile(speedFilePath) + if err != nil { + log.Log.Error(err, "GetNetDevLinkSpeed(): fail to read Link Speed file", "path", speedFilePath) + return "" + } + + return fmt.Sprintf("%s Mb/s", strings.TrimSpace(string(data))) +} diff --git a/pkg/host/service.go b/pkg/host/service.go new file mode 100644 index 000000000..513542e66 --- /dev/null +++ b/pkg/host/service.go @@ -0,0 +1,313 @@ +package host + +import ( + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/coreos/go-systemd/v22/unit" + "gopkg.in/yaml.v3" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +// TODO: handle this to support unit-tests +const systemdDir = "/usr/lib/systemd/system/" + +var ( + // Remove run condition form the service + ConditionOpt = &unit.UnitOption{ + Section: "Unit", + Name: "ConditionPathExists", + Value: "!/etc/ignition-machine-config-encapsulated.json", + } +) + +type ServiceInterface interface { + // IsServiceExist checks if the requested systemd service exist on the system + IsServiceExist(string) (bool, error) + // IsServiceEnabled checks if the requested systemd service is enabled on the system + IsServiceEnabled(string) (bool, error) + // ReadService reads a systemd servers and return it as a struct + ReadService(string) (*Service, error) + // EnableService enables a systemd server on the host + EnableService(service *Service) error + // ReadServiceManifestFile reads the systemd manifest for a specific service + ReadServiceManifestFile(path string) (*Service, error) + // RemoveFromService removes a systemd service from the host + RemoveFromService(service *Service, options ...*unit.UnitOption) (*Service, error) + // ReadScriptManifestFile reads the script manifest from a systemd service + ReadScriptManifestFile(path string) (*ScriptManifestFile, error) + // ReadServiceInjectionManifestFile reads the injection manifest file for the systemd service + ReadServiceInjectionManifestFile(path string) (*Service, error) + // CompareServices compare two servers and return true if they are equal + CompareServices(serviceA, serviceB *Service) (bool, error) + // UpdateSystemService updates a system service on the host + UpdateSystemService(serviceObj *Service) error +} + +type service struct { + utilsHelper utils.CmdInterface +} + +func newServiceInterface(utilsHelper utils.CmdInterface) ServiceInterface { + return &service{utilsHelper: utilsHelper} +} + +type Service struct { + Name string + Path string + Content string +} + +// ServiceInjectionManifestFile service injection manifest file structure +type ServiceInjectionManifestFile struct { + Name string + Dropins []struct { + Contents string + } +} + +// ServiceManifestFile service manifest file structure +type ServiceManifestFile struct { + Name string + Contents string +} + +// ScriptManifestFile script manifest file structure +type ScriptManifestFile struct { + Path string + Contents struct { + Inline string + } +} + +// IsServiceExist check if service unit exist +func (s *service) IsServiceExist(servicePath string) (bool, error) { + _, err := os.Stat(path.Join(consts.Chroot, servicePath)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +// IsServiceEnabled check if service exist and enabled +func (s *service) IsServiceEnabled(servicePath string) (bool, error) { + exist, err := s.IsServiceExist(servicePath) + if err != nil || !exist { + return false, err + } + serviceName := filepath.Base(servicePath) + // Change root dir + exit, err := s.utilsHelper.Chroot(consts.Chroot) + if err != nil { + return false, err + } + defer exit() + + // TODO: add check for the output and logs + _, _, err = s.utilsHelper.RunCommand("systemctl", "is-enabled", serviceName) + return err == nil, nil +} + +// ReadService read service from given path +func (s *service) ReadService(servicePath string) (*Service, error) { + data, err := os.ReadFile(path.Join(consts.Chroot, servicePath)) + if err != nil { + return nil, err + } + + return &Service{ + Name: filepath.Base(servicePath), + Path: servicePath, + Content: string(data), + }, nil +} + +// EnableService creates service file and enables it with systemctl enable +func (s *service) EnableService(service *Service) error { + // Write service file + err := os.WriteFile(path.Join(consts.Chroot, service.Path), []byte(service.Content), 0644) + if err != nil { + return err + } + + // Change root dir + exit, err := s.utilsHelper.Chroot(consts.Chroot) + if err != nil { + return err + } + defer exit() + + // Enable service + _, _, err = s.utilsHelper.RunCommand("systemctl", "enable", service.Name) + return err +} + +// CompareServices compare 2 service and return true if serviceA has all the fields of serviceB +func (s *service) CompareServices(serviceA, serviceB *Service) (bool, error) { + optsA, err := unit.Deserialize(strings.NewReader(serviceA.Content)) + if err != nil { + return false, err + } + optsB, err := unit.Deserialize(strings.NewReader(serviceB.Content)) + if err != nil { + return false, err + } + +OUTER: + for _, optB := range optsB { + for _, optA := range optsA { + if optA.Match(optB) { + continue OUTER + } + } + log.Log.V(2).Info("CompareServices", "ServiceA", optsA, "ServiceB", *optB) + return true, nil + } + + return false, nil +} + +// RemoveFromService removes given fields from service +func (s *service) RemoveFromService(service *Service, options ...*unit.UnitOption) (*Service, error) { + opts, err := unit.Deserialize(strings.NewReader(service.Content)) + if err != nil { + return nil, err + } + + var newServiceOptions []*unit.UnitOption +OUTER: + for _, opt := range opts { + for _, optRemove := range options { + if opt.Match(optRemove) { + continue OUTER + } + } + + newServiceOptions = append(newServiceOptions, opt) + } + + data, err := io.ReadAll(unit.Serialize(newServiceOptions)) + if err != nil { + return nil, err + } + + return &Service{ + Name: service.Name, + Path: service.Path, + Content: string(data), + }, nil +} + +// ReadServiceInjectionManifestFile reads service injection file +func (s *service) ReadServiceInjectionManifestFile(path string) (*Service, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var serviceContent ServiceInjectionManifestFile + if err := yaml.Unmarshal(data, &serviceContent); err != nil { + return nil, err + } + + return &Service{ + Name: serviceContent.Name, + Path: systemdDir + serviceContent.Name, + Content: serviceContent.Dropins[0].Contents, + }, nil +} + +// ReadServiceManifestFile reads service file +func (s *service) ReadServiceManifestFile(path string) (*Service, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var serviceFile *ServiceManifestFile + if err := yaml.Unmarshal(data, &serviceFile); err != nil { + return nil, err + } + + return &Service{ + Name: serviceFile.Name, + Path: "/etc/systemd/system/" + serviceFile.Name, + Content: serviceFile.Contents, + }, nil +} + +// ReadScriptManifestFile reads script file +func (s *service) ReadScriptManifestFile(path string) (*ScriptManifestFile, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var scriptFile *ScriptManifestFile + if err := yaml.Unmarshal(data, &scriptFile); err != nil { + return nil, err + } + + return scriptFile, nil +} + +func (s *service) UpdateSystemService(serviceObj *Service) error { + systemService, err := s.ReadService(serviceObj.Path) + if err != nil { + return err + } + if systemService == nil { + // Invalid case to reach here + return fmt.Errorf("can't update non-existing service %q", serviceObj.Name) + } + serviceOptions, err := unit.Deserialize(strings.NewReader(serviceObj.Content)) + if err != nil { + return err + } + updatedService, err := appendToService(systemService, serviceOptions...) + if err != nil { + return err + } + + return s.EnableService(updatedService) +} + +// appendToService appends given fields to service +func appendToService(service *Service, options ...*unit.UnitOption) (*Service, error) { + serviceOptions, err := unit.Deserialize(strings.NewReader(service.Content)) + if err != nil { + return nil, err + } + +OUTER: + for _, appendOpt := range options { + for _, opt := range serviceOptions { + if opt.Match(appendOpt) { + continue OUTER + } + } + serviceOptions = append(serviceOptions, appendOpt) + } + + data, err := io.ReadAll(unit.Serialize(serviceOptions)) + if err != nil { + return nil, err + } + + return &Service{ + Name: service.Name, + Path: service.Path, + Content: string(data), + }, nil +} diff --git a/pkg/host/sriov.go b/pkg/host/sriov.go new file mode 100644 index 000000000..7d6343771 --- /dev/null +++ b/pkg/host/sriov.go @@ -0,0 +1,629 @@ +package host + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/jaypipes/ghw" + "github.com/vishvananda/netlink" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/log" + + dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" +) + +type SriovInterface interface { + // SetSriovNumVfs changes the number of virtual functions allocated for a specific + // physical function base on pci address + SetSriovNumVfs(string, int) error + // GetVfInfo returns the virtual function information is the operator struct from the host information + GetVfInfo(string, []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction + // SetVfGUID sets the GUID for a virtual function + SetVfGUID(string, netlink.Link) error + // VFIsReady returns the interface virtual function if the device is ready + VFIsReady(string) (netlink.Link, error) + // SetVfAdminMac sets the virtual function administrative mac address via the physical function + SetVfAdminMac(string, netlink.Link, netlink.Link) error + // GetNicSriovMode returns the interface mode + // supported modes SR-IOV legacy and switchdev + GetNicSriovMode(string) (string, error) + // GetLinkType return the link type + // supported types are ethernet and infiniband + GetLinkType(sriovnetworkv1.InterfaceExt) string + // ResetSriovDevice resets the number of virtual function for the specific physical function to zero + ResetSriovDevice(sriovnetworkv1.InterfaceExt) error + // DiscoverSriovDevices returns a list of all the available SR-IOV capable network interfaces on the system + DiscoverSriovDevices(StoreManagerInterface) ([]sriovnetworkv1.InterfaceExt, error) + // ConfigSriovDevice configure the request SR-IOV device with the desired configuration + ConfigSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) error + // ConfigSriovInterfaces configure multiple SR-IOV devices with the desired configuration + ConfigSriovInterfaces(StoreManagerInterface, []sriovnetworkv1.Interface, []sriovnetworkv1.InterfaceExt, map[string]bool) error + // ConfigSriovInterfaces configure virtual functions for virtual environments with the desired configuration + ConfigSriovDeviceVirtual(iface *sriovnetworkv1.Interface) error +} + +type sriov struct { + utilsHelper utils.CmdInterface + kernelHelper KernelInterface + networkHelper NetworkInterface + udevHelper UdevInterface +} + +func newSriovInterface(utilsHelper utils.CmdInterface, + kernelHelper KernelInterface, + networkHelper NetworkInterface, + udevHelper UdevInterface) SriovInterface { + return &sriov{utilsHelper: utilsHelper, kernelHelper: kernelHelper, networkHelper: networkHelper, udevHelper: udevHelper} +} + +func (s *sriov) SetSriovNumVfs(pciAddr string, numVfs int) error { + log.Log.V(2).Info("SetSriovNumVfs(): set NumVfs", "device", pciAddr, "numVfs", numVfs) + numVfsFilePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, consts.NumVfsFile) + bs := []byte(strconv.Itoa(numVfs)) + err := os.WriteFile(numVfsFilePath, []byte("0"), os.ModeAppend) + if err != nil { + log.Log.Error(err, "SetSriovNumVfs(): fail to reset NumVfs file", "path", numVfsFilePath) + return err + } + err = os.WriteFile(numVfsFilePath, bs, os.ModeAppend) + if err != nil { + log.Log.Error(err, "SetSriovNumVfs(): fail to set NumVfs file", "path", numVfsFilePath) + return err + } + return nil +} + +func (s *sriov) ResetSriovDevice(ifaceStatus sriovnetworkv1.InterfaceExt) error { + log.Log.V(2).Info("ResetSriovDevice(): reset SRIOV device", "address", ifaceStatus.PciAddress) + if err := s.SetSriovNumVfs(ifaceStatus.PciAddress, 0); err != nil { + return err + } + if ifaceStatus.LinkType == consts.LinkTypeETH { + var mtu int + is := sriovnetworkv1.InitialState.GetInterfaceStateByPciAddress(ifaceStatus.PciAddress) + if is != nil { + mtu = is.Mtu + } else { + mtu = 1500 + } + log.Log.V(2).Info("ResetSriovDevice(): reset mtu", "value", mtu) + if err := s.networkHelper.SetNetdevMTU(ifaceStatus.PciAddress, mtu); err != nil { + return err + } + } else if ifaceStatus.LinkType == consts.LinkTypeIB { + if err := s.networkHelper.SetNetdevMTU(ifaceStatus.PciAddress, 2048); err != nil { + return err + } + } + return nil +} + +func (s *sriov) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { + driver, err := dputils.GetDriverName(pciAddr) + if err != nil { + log.Log.Error(err, "getVfInfo(): unable to parse device driver", "device", pciAddr) + } + id, err := dputils.GetVFID(pciAddr) + if err != nil { + log.Log.Error(err, "getVfInfo(): unable to get VF index", "device", pciAddr) + } + vf := sriovnetworkv1.VirtualFunction{ + PciAddress: pciAddr, + Driver: driver, + VfID: id, + } + + if mtu := s.networkHelper.GetNetdevMTU(pciAddr); mtu > 0 { + vf.Mtu = mtu + } + if name := s.networkHelper.TryGetInterfaceName(pciAddr); name != "" { + vf.Name = name + vf.Mac = s.networkHelper.GetNetDevMac(name) + } + + for _, device := range devices { + if pciAddr == device.Address { + vf.Vendor = device.Vendor.ID + vf.DeviceID = device.Product.ID + break + } + continue + } + return vf +} + +func (s *sriov) SetVfGUID(vfAddr string, pfLink netlink.Link) error { + log.Log.Info("SetVfGUID()", "vf", vfAddr) + vfID, err := dputils.GetVFID(vfAddr) + if err != nil { + log.Log.Error(err, "SetVfGUID(): unable to get VF id", "address", vfAddr) + return err + } + guid := utils.GenerateRandomGUID() + if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { + return err + } + if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + return err + } + if err = s.kernelHelper.Unbind(vfAddr); err != nil { + return err + } + + return nil +} + +func (s *sriov) VFIsReady(pciAddr string) (netlink.Link, error) { + log.Log.Info("VFIsReady()", "device", pciAddr) + var err error + var vfLink netlink.Link + err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { + vfName := s.networkHelper.TryGetInterfaceName(pciAddr) + vfLink, err = netlink.LinkByName(vfName) + if err != nil { + log.Log.Error(err, "VFIsReady(): unable to get VF link", "device", pciAddr) + } + return err == nil, nil + }) + if err != nil { + return vfLink, err + } + return vfLink, nil +} + +func (s *sriov) SetVfAdminMac(vfAddr string, pfLink, vfLink netlink.Link) error { + log.Log.Info("SetVfAdminMac()", "vf", vfAddr) + + vfID, err := dputils.GetVFID(vfAddr) + if err != nil { + log.Log.Error(err, "SetVfAdminMac(): unable to get VF id", "address", vfAddr) + return err + } + + if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { + return err + } + + return nil +} + +func (s *sriov) DiscoverSriovDevices(storeManager StoreManagerInterface) ([]sriovnetworkv1.InterfaceExt, error) { + log.Log.V(2).Info("DiscoverSriovDevices") + pfList := []sriovnetworkv1.InterfaceExt{} + + pci, err := ghw.PCI() + if err != nil { + return nil, fmt.Errorf("DiscoverSriovDevices(): error getting PCI info: %v", err) + } + + devices := pci.ListDevices() + if len(devices) == 0 { + return nil, fmt.Errorf("DiscoverSriovDevices(): could not retrieve PCI devices") + } + + for _, device := range devices { + devClass, err := strconv.ParseInt(device.Class.ID, 16, 64) + if err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): unable to parse device class, skipping", + "device", device) + continue + } + if devClass != consts.NetClass { + // Not network device + continue + } + + // TODO: exclude devices used by host system + + if dputils.IsSriovVF(device.Address) { + continue + } + + driver, err := dputils.GetDriverName(device.Address) + if err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): unable to parse device driver for device, skipping", "device", device) + continue + } + + deviceNames, err := dputils.GetNetNames(device.Address) + if err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): unable to get device names for device, skipping", "device", device) + continue + } + + if len(deviceNames) == 0 { + // no network devices found, skipping device + continue + } + + if !vars.DevMode { + if !sriovnetworkv1.IsSupportedModel(device.Vendor.ID, device.Product.ID) { + log.Log.Info("DiscoverSriovDevices(): unsupported device", "device", device) + continue + } + } + + iface := sriovnetworkv1.InterfaceExt{ + PciAddress: device.Address, + Driver: driver, + Vendor: device.Vendor.ID, + DeviceID: device.Product.ID, + } + if mtu := s.networkHelper.GetNetdevMTU(device.Address); mtu > 0 { + iface.Mtu = mtu + } + if name := s.networkHelper.TryGetInterfaceName(device.Address); name != "" { + iface.Name = name + iface.Mac = s.networkHelper.GetNetDevMac(name) + iface.LinkSpeed = s.networkHelper.GetNetDevLinkSpeed(name) + } + iface.LinkType = s.GetLinkType(iface) + + pfStatus, exist, err := storeManager.LoadPfsStatus(iface.PciAddress) + if err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): failed to load PF status from disk") + } else { + if exist { + iface.ExternallyManaged = pfStatus.ExternallyManaged + } + } + + if dputils.IsSriovPF(device.Address) { + iface.TotalVfs = dputils.GetSriovVFcapacity(device.Address) + iface.NumVfs = dputils.GetVFconfigured(device.Address) + if iface.EswitchMode, err = s.GetNicSriovMode(device.Address); err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): warning, unable to get device eswitch mode", + "device", device.Address) + } + if dputils.SriovConfigured(device.Address) { + vfs, err := dputils.GetVFList(device.Address) + if err != nil { + log.Log.Error(err, "DiscoverSriovDevices(): unable to parse VFs for device, skipping", + "device", device) + continue + } + for _, vf := range vfs { + instance := s.GetVfInfo(vf, devices) + iface.VFs = append(iface.VFs, instance) + } + } + } + pfList = append(pfList, iface) + } + + return pfList, nil +} + +func (s *sriov) ConfigSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) error { + log.Log.V(2).Info("configSriovDevice(): configure sriov device", + "device", iface.PciAddress, "config", iface) + var err error + if iface.NumVfs > ifaceStatus.TotalVfs { + err := fmt.Errorf("cannot config SRIOV device: NumVfs (%d) is larger than TotalVfs (%d)", iface.NumVfs, ifaceStatus.TotalVfs) + log.Log.Error(err, "configSriovDevice(): fail to set NumVfs for device", "device", iface.PciAddress) + return err + } + // set numVFs + if iface.NumVfs != ifaceStatus.NumVfs { + if iface.ExternallyManaged { + if iface.NumVfs > ifaceStatus.NumVfs { + errMsg := fmt.Sprintf("configSriovDevice(): number of request virtual functions %d is not equal to configured virtual functions %d but the policy is configured as ExternallyManaged for device %s", iface.NumVfs, ifaceStatus.NumVfs, iface.PciAddress) + log.Log.Error(nil, errMsg) + return fmt.Errorf(errMsg) + } + } else { + // create the udev rule to disable all the vfs from network manager as this vfs are managed by the operator + err = s.udevHelper.AddUdevRule(iface.PciAddress) + if err != nil { + return err + } + + err = s.SetSriovNumVfs(iface.PciAddress, iface.NumVfs) + if err != nil { + log.Log.Error(err, "configSriovDevice(): fail to set NumVfs for device", "device", iface.PciAddress) + errRemove := s.udevHelper.RemoveUdevRule(iface.PciAddress) + if errRemove != nil { + log.Log.Error(errRemove, "configSriovDevice(): fail to remove udev rule", "device", iface.PciAddress) + } + return err + } + } + } + // set PF mtu + if iface.Mtu > 0 && iface.Mtu > ifaceStatus.Mtu { + err = s.networkHelper.SetNetdevMTU(iface.PciAddress, iface.Mtu) + if err != nil { + log.Log.Error(err, "configSriovDevice(): fail to set mtu for PF", "device", iface.PciAddress) + return err + } + } + // Config VFs + if iface.NumVfs > 0 { + vfAddrs, err := dputils.GetVFList(iface.PciAddress) + if err != nil { + log.Log.Error(err, "configSriovDevice(): unable to parse VFs for device", "device", iface.PciAddress) + } + pfLink, err := netlink.LinkByName(iface.Name) + if err != nil { + log.Log.Error(err, "configSriovDevice(): unable to get PF link for device", "device", iface) + return err + } + + for _, addr := range vfAddrs { + var group *sriovnetworkv1.VfGroup + + vfID, err := dputils.GetVFID(addr) + if err != nil { + log.Log.Error(err, "configSriovDevice(): unable to get VF id", "device", iface.PciAddress) + return err + } + + for i := range iface.VfGroups { + if sriovnetworkv1.IndexInRange(vfID, iface.VfGroups[i].VfRange) { + group = &iface.VfGroups[i] + break + } + } + + // VF group not found. + if group == nil { + continue + } + + // only set GUID and MAC for VF with default driver + // for userspace drivers like vfio we configure the vf mac using the kernel nic mac address + // before we switch to the userspace driver + if yes, d := s.kernelHelper.HasDriver(addr); yes && !sriovnetworkv1.StringInArray(d, vars.DpdkDrivers) { + // LinkType is an optional field. Let's fallback to current link type + // if nothing is specified in the SriovNodePolicy + linkType := iface.LinkType + if linkType == "" { + linkType = ifaceStatus.LinkType + } + if strings.EqualFold(linkType, consts.LinkTypeIB) { + if err = s.SetVfGUID(addr, pfLink); err != nil { + return err + } + } else { + vfLink, err := s.VFIsReady(addr) + if err != nil { + log.Log.Error(err, "configSriovDevice(): VF link is not ready", "address", addr) + err = s.kernelHelper.RebindVfToDefaultDriver(addr) + if err != nil { + log.Log.Error(err, "configSriovDevice(): failed to rebind VF", "address", addr) + return err + } + + // Try to check the VF status again + vfLink, err = s.VFIsReady(addr) + if err != nil { + log.Log.Error(err, "configSriovDevice(): VF link is not ready", "address", addr) + return err + } + } + if err = s.SetVfAdminMac(addr, pfLink, vfLink); err != nil { + log.Log.Error(err, "configSriovDevice(): fail to configure VF admin mac", "device", addr) + return err + } + } + } + + if err = s.kernelHelper.UnbindDriverIfNeeded(addr, group.IsRdma); err != nil { + return err + } + + if !sriovnetworkv1.StringInArray(group.DeviceType, vars.DpdkDrivers) { + if err := s.kernelHelper.BindDefaultDriver(addr); err != nil { + log.Log.Error(err, "configSriovDevice(): fail to bind default driver for device", "device", addr) + return err + } + // only set MTU for VF with default driver + if group.Mtu > 0 { + if err := s.networkHelper.SetNetdevMTU(addr, group.Mtu); err != nil { + log.Log.Error(err, "configSriovDevice(): fail to set mtu for VF", "address", addr) + return err + } + } + } else { + if err := s.kernelHelper.BindDpdkDriver(addr, group.DeviceType); err != nil { + log.Log.Error(err, "configSriovDevice(): fail to bind driver for device", + "driver", group.DeviceType, "device", addr) + return err + } + } + } + } + // Set PF link up + pfLink, err := netlink.LinkByName(ifaceStatus.Name) + if err != nil { + return err + } + if pfLink.Attrs().OperState != netlink.OperUp { + err = netlink.LinkSetUp(pfLink) + if err != nil { + return err + } + } + return nil +} + +func (s *sriov) ConfigSriovInterfaces(storeManager StoreManagerInterface, interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error { + if s.kernelHelper.IsKernelLockdownMode() && mlx.HasMellanoxInterfacesInSpec(ifaceStatuses, interfaces) { + log.Log.Error(nil, "cannot use mellanox devices when in kernel lockdown mode") + return fmt.Errorf("cannot use mellanox devices when in kernel lockdown mode") + } + + for _, ifaceStatus := range ifaceStatuses { + configured := false + for _, iface := range interfaces { + if iface.PciAddress == ifaceStatus.PciAddress { + configured = true + + if skip := pfsToConfig[iface.PciAddress]; skip { + break + } + + if !sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) { + log.Log.V(2).Info("syncNodeState(): no need update interface", "address", iface.PciAddress) + + // Save the PF status to the host + err := storeManager.SaveLastPfAppliedStatus(&iface) + if err != nil { + log.Log.Error(err, "SyncNodeState(): failed to save PF applied config to host") + return err + } + + break + } + if err := s.ConfigSriovDevice(&iface, &ifaceStatus); err != nil { + log.Log.Error(err, "SyncNodeState(): fail to configure sriov interface. resetting interface.", "address", iface.PciAddress) + if iface.ExternallyManaged { + log.Log.Info("SyncNodeState(): skipping device reset as the nic is marked as externally created") + } else { + if resetErr := s.ResetSriovDevice(ifaceStatus); resetErr != nil { + log.Log.Error(resetErr, "SyncNodeState(): failed to reset on error SR-IOV interface") + } + } + return err + } + + // Save the PF status to the host + err := storeManager.SaveLastPfAppliedStatus(&iface) + if err != nil { + log.Log.Error(err, "SyncNodeState(): failed to save PF applied config to host") + return err + } + break + } + } + if !configured && ifaceStatus.NumVfs > 0 { + if skip := pfsToConfig[ifaceStatus.PciAddress]; skip { + continue + } + + // load the PF info + pfStatus, exist, err := storeManager.LoadPfsStatus(ifaceStatus.PciAddress) + if err != nil { + log.Log.Error(err, "SyncNodeState(): failed to load info about PF status for device", + "address", ifaceStatus.PciAddress) + return err + } + + if !exist { + log.Log.Info("SyncNodeState(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping the device reset", + "pf-name", ifaceStatus.Name, + "address", ifaceStatus.PciAddress) + continue + } + + if pfStatus.ExternallyManaged { + log.Log.Info("SyncNodeState(): PF name with pci address was externally created skipping the device reset", + "pf-name", ifaceStatus.Name, + "address", ifaceStatus.PciAddress) + continue + } else { + err = s.udevHelper.RemoveUdevRule(ifaceStatus.PciAddress) + if err != nil { + return err + } + } + + if err = s.ResetSriovDevice(ifaceStatus); err != nil { + return err + } + } + } + return nil +} + +func (s *sriov) ConfigSriovDeviceVirtual(iface *sriovnetworkv1.Interface) error { + log.Log.V(2).Info("ConfigSriovDeviceVirtual(): config interface", "address", iface.PciAddress, "config", iface) + // Config VFs + if iface.NumVfs > 0 { + if iface.NumVfs > 1 { + log.Log.Error(nil, "ConfigSriovDeviceVirtual(): in a virtual environment, only one VF per interface", + "numVfs", iface.NumVfs) + return errors.New("NumVfs > 1") + } + if len(iface.VfGroups) != 1 { + log.Log.Error(nil, "ConfigSriovDeviceVirtual(): missing VFGroup") + return errors.New("NumVfs != 1") + } + addr := iface.PciAddress + log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "address", addr) + driver := "" + vfID := 0 + for _, group := range iface.VfGroups { + log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "group", group) + if sriovnetworkv1.IndexInRange(vfID, group.VfRange) { + log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "indexInRange", vfID) + if sriovnetworkv1.StringInArray(group.DeviceType, vars.DpdkDrivers) { + log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "driver", group.DeviceType) + driver = group.DeviceType + } + break + } + } + if driver == "" { + log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind default") + if err := s.kernelHelper.BindDefaultDriver(addr); err != nil { + log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind default driver", "device", addr) + return err + } + } else { + log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind driver", "driver", driver) + if err := s.kernelHelper.BindDpdkDriver(addr, driver); err != nil { + log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind driver for device", + "driver", driver, "device", addr) + return err + } + } + } + return nil +} + +func (s *sriov) GetNicSriovMode(pciAddress string) (string, error) { + log.Log.V(2).Info("GetNicSriovMode()", "device", pciAddress) + + devLink, err := netlink.DevLinkGetDeviceByName("pci", pciAddress) + if err != nil { + if errors.Is(err, syscall.ENODEV) { + // the device doesn't support devlink + return "", nil + } + return "", err + } + + return devLink.Attrs.Eswitch.Mode, nil +} + +func (s *sriov) GetLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string { + log.Log.V(2).Info("GetLinkType()", "device", ifaceStatus.PciAddress) + if ifaceStatus.Name != "" { + link, err := netlink.LinkByName(ifaceStatus.Name) + if err != nil { + log.Log.Error(err, "GetLinkType(): failed to get link", "device", ifaceStatus.Name) + return "" + } + linkType := link.Attrs().EncapType + if linkType == "ether" { + return consts.LinkTypeETH + } else if linkType == "infiniband" { + return consts.LinkTypeIB + } + } + + return "" +} diff --git a/pkg/utils/store.go b/pkg/host/store.go similarity index 55% rename from pkg/utils/store.go rename to pkg/host/store.go index dd1b44626..1bced8c9d 100644 --- a/pkg/utils/store.go +++ b/pkg/host/store.go @@ -1,20 +1,17 @@ -package utils +package host import ( "encoding/json" "fmt" "os" - "path" "path/filepath" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -const ( - SriovConfBasePath = "/etc/sriov-operator" - PfAppliedConfig = SriovConfBasePath + "/pci" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) // Contains all the file storing on the host @@ -24,27 +21,29 @@ type StoreManagerInterface interface { ClearPCIAddressFolder() error SaveLastPfAppliedStatus(PfInfo *sriovnetworkv1.Interface) error LoadPfsStatus(pciAddress string) (*sriovnetworkv1.Interface, bool, error) + + GetCheckPointNodeState() (*sriovnetworkv1.SriovNetworkNodeState, error) + WriteCheckpointFile(*sriovnetworkv1.SriovNetworkNodeState) error } -type StoreManager struct { - RunOnHost bool +type storeManager struct { } // NewStoreManager: create the initial folders needed to store the info about the PF // and return a storeManager struct that implements the StoreManagerInterface interface -func NewStoreManager(runOnHost bool) (StoreManagerInterface, error) { - if err := createOperatorConfigFolderIfNeeded(runOnHost); err != nil { +func NewStoreManager() (StoreManagerInterface, error) { + if err := createOperatorConfigFolderIfNeeded(); err != nil { return nil, err } - return &StoreManager{runOnHost}, nil + return &storeManager{}, nil } // createOperatorConfigFolderIfNeeded: create the operator base folder on the host // together with the pci folder to save the PF status objects as json files -func createOperatorConfigFolderIfNeeded(runOnHost bool) error { - hostExtension := getHostExtension(runOnHost) - SriovConfBasePathUse := filepath.Join(hostExtension, SriovConfBasePath) +func createOperatorConfigFolderIfNeeded() error { + hostExtension := utils.GetHostExtension() + SriovConfBasePathUse := filepath.Join(hostExtension, consts.SriovConfBasePath) _, err := os.Stat(SriovConfBasePathUse) if err != nil { if os.IsNotExist(err) { @@ -57,7 +56,7 @@ func createOperatorConfigFolderIfNeeded(runOnHost bool) error { } } - PfAppliedConfigUse := filepath.Join(hostExtension, PfAppliedConfig) + PfAppliedConfigUse := filepath.Join(hostExtension, consts.PfAppliedConfig) _, err = os.Stat(PfAppliedConfigUse) if err != nil { if os.IsNotExist(err) { @@ -74,9 +73,9 @@ func createOperatorConfigFolderIfNeeded(runOnHost bool) error { } // ClearPCIAddressFolder: removes all the PFs storage information -func (s *StoreManager) ClearPCIAddressFolder() error { - hostExtension := getHostExtension(s.RunOnHost) - PfAppliedConfigUse := filepath.Join(hostExtension, PfAppliedConfig) +func (s *storeManager) ClearPCIAddressFolder() error { + hostExtension := utils.GetHostExtension() + PfAppliedConfigUse := filepath.Join(hostExtension, consts.PfAppliedConfig) _, err := os.Stat(PfAppliedConfigUse) if err != nil { if os.IsNotExist(err) { @@ -100,24 +99,24 @@ func (s *StoreManager) ClearPCIAddressFolder() error { // SaveLastPfAppliedStatus will save the PF object as a json into the /etc/sriov-operator/pci/ // this function must be called after running the chroot function -func (s *StoreManager) SaveLastPfAppliedStatus(PfInfo *sriovnetworkv1.Interface) error { +func (s *storeManager) SaveLastPfAppliedStatus(PfInfo *sriovnetworkv1.Interface) error { data, err := json.Marshal(PfInfo) if err != nil { log.Log.Error(err, "failed to marshal PF status", "status", *PfInfo) return err } - hostExtension := getHostExtension(s.RunOnHost) - pathFile := filepath.Join(hostExtension, PfAppliedConfig, PfInfo.PciAddress) + hostExtension := utils.GetHostExtension() + pathFile := filepath.Join(hostExtension, consts.PfAppliedConfig, PfInfo.PciAddress) err = os.WriteFile(pathFile, data, 0644) return err } // LoadPfsStatus convert the /etc/sriov-operator/pci/ json to pfstatus // returns false if the file doesn't exist. -func (s *StoreManager) LoadPfsStatus(pciAddress string) (*sriovnetworkv1.Interface, bool, error) { - hostExtension := getHostExtension(s.RunOnHost) - pathFile := filepath.Join(hostExtension, PfAppliedConfig, pciAddress) +func (s *storeManager) LoadPfsStatus(pciAddress string) (*sriovnetworkv1.Interface, bool, error) { + hostExtension := utils.GetHostExtension() + pathFile := filepath.Join(hostExtension, consts.PfAppliedConfig, pciAddress) pfStatus := &sriovnetworkv1.Interface{} data, err := os.ReadFile(pathFile) if err != nil { @@ -137,9 +136,45 @@ func (s *StoreManager) LoadPfsStatus(pciAddress string) (*sriovnetworkv1.Interfa return pfStatus, true, nil } -func getHostExtension(runOnHost bool) string { - if !runOnHost { - return path.Join(FilesystemRoot, "/host") +func (s *storeManager) GetCheckPointNodeState() (*sriovnetworkv1.SriovNetworkNodeState, error) { + log.Log.Info("getCheckPointNodeState()") + configdir := filepath.Join(vars.Destdir, consts.CheckpointFileName) + file, err := os.OpenFile(configdir, os.O_RDONLY, 0644) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer file.Close() + if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { + return nil, err + } + + return &sriovnetworkv1.InitialState, nil +} + +func (s *storeManager) WriteCheckpointFile(ns *sriovnetworkv1.SriovNetworkNodeState) error { + configdir := filepath.Join(vars.Destdir, consts.CheckpointFileName) + file, err := os.OpenFile(configdir, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return err } - return FilesystemRoot + defer file.Close() + log.Log.Info("WriteCheckpointFile(): try to decode the checkpoint file") + if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { + log.Log.V(2).Error(err, "WriteCheckpointFile(): fail to decode, writing new file instead") + log.Log.Info("WriteCheckpointFile(): write checkpoint file") + if err = file.Truncate(0); err != nil { + return err + } + if _, err = file.Seek(0, 0); err != nil { + return err + } + if err = json.NewEncoder(file).Encode(*ns); err != nil { + return err + } + sriovnetworkv1.InitialState = *ns + } + return nil } diff --git a/pkg/host/udev.go b/pkg/host/udev.go new file mode 100644 index 000000000..5f4daee58 --- /dev/null +++ b/pkg/host/udev.go @@ -0,0 +1,190 @@ +package host + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +type UdevInterface interface { + // WriteSwitchdevConfFile writes the needed switchdev configuration files for HW offload support + WriteSwitchdevConfFile(*sriovnetworkv1.SriovNetworkNodeState, map[string]bool) (bool, error) + // PrepareNMUdevRule creates the needed udev rules to disable NetworkManager from + // our managed SR-IOV virtual functions + PrepareNMUdevRule([]string) error + // AddUdevRule adds a specific udev rule to the system + AddUdevRule(string) error + // RemoveUdevRule removes a udev rule from the system + RemoveUdevRule(string) error +} + +type udev struct { + utilsHelper utils.CmdInterface +} + +func newUdevInterface(utilsHelper utils.CmdInterface) UdevInterface { + return &udev{utilsHelper: utilsHelper} +} + +type config struct { + Interfaces []sriovnetworkv1.Interface `json:"interfaces"` +} + +func (u *udev) PrepareNMUdevRule(supportedVfIds []string) error { + log.Log.V(2).Info("PrepareNMUdevRule()") + filePath := filepath.Join(vars.FilesystemRoot, consts.HostUdevRulesFolder, "10-nm-unmanaged.rules") + + // remove the old unmanaged rules file + if _, err := os.Stat(filePath); err == nil { + err = os.Remove(filePath) + if err != nil { + log.Log.Error(err, "failed to remove the network manager global unmanaged rule", + "path", filePath) + } + } + + // create the pf finder script for udev rules + stdout, stderr, err := u.utilsHelper.RunCommand("/bin/bash", filepath.Join(vars.FilesystemRoot, consts.UdevDisableNM)) + if err != nil { + log.Log.Error(err, "PrepareNMUdevRule(): failed to prepare nmUdevRule", "stderr", stderr) + return err + } + log.Log.V(2).Info("PrepareNMUdevRule()", "stdout", stdout) + + //save the device list to use for udev rules + vars.SupportedVfIds = supportedVfIds + return nil +} + +func (u *udev) WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToSkip map[string]bool) (bool, error) { + cfg := config{} + for _, iface := range newState.Spec.Interfaces { + for _, ifaceStatus := range newState.Status.Interfaces { + if iface.PciAddress != ifaceStatus.PciAddress { + continue + } + + if skip := pfsToSkip[iface.PciAddress]; !skip { + continue + } + + if iface.NumVfs > 0 { + var vfGroups []sriovnetworkv1.VfGroup = nil + ifc, err := sriovnetworkv1.FindInterface(newState.Spec.Interfaces, iface.Name) + if err != nil { + log.Log.Error(err, "WriteSwitchdevConfFile(): fail find interface") + } else { + vfGroups = ifc.VfGroups + } + i := sriovnetworkv1.Interface{ + // Not passing all the contents, since only NumVfs and EswitchMode can be configured by configure-switchdev.sh currently. + Name: iface.Name, + PciAddress: iface.PciAddress, + NumVfs: iface.NumVfs, + Mtu: iface.Mtu, + VfGroups: vfGroups, + } + + if iface.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { + i.EswitchMode = iface.EswitchMode + } + cfg.Interfaces = append(cfg.Interfaces, i) + } + } + } + _, err := os.Stat(consts.SriovHostSwitchDevConfPath) + if err != nil { + if os.IsNotExist(err) { + if len(cfg.Interfaces) == 0 { + return false, nil + } + + // TODO: refactor this function to allow using vars.FilesystemRoot for unit-tests + // Create the sriov-operator folder on the host if it doesn't exist + if _, err := os.Stat(consts.Host + consts.SriovConfBasePath); os.IsNotExist(err) { + err = os.Mkdir(consts.Host+consts.SriovConfBasePath, os.ModeDir) + if err != nil { + log.Log.Error(err, "WriteConfFile(): failed to create sriov-operator folder") + return false, err + } + } + + log.Log.V(2).Info("WriteSwitchdevConfFile(): file not existed, create it") + _, err = os.Create(consts.SriovHostSwitchDevConfPath) + if err != nil { + log.Log.Error(err, "WriteSwitchdevConfFile(): failed to create file") + return false, err + } + } else { + return false, err + } + } + oldContent, err := os.ReadFile(consts.SriovHostSwitchDevConfPath) + if err != nil { + log.Log.Error(err, "WriteSwitchdevConfFile(): failed to read file") + return false, err + } + var newContent []byte + if len(cfg.Interfaces) != 0 { + newContent, err = json.Marshal(cfg) + if err != nil { + log.Log.Error(err, "WriteSwitchdevConfFile(): fail to marshal config") + return false, err + } + } + + if bytes.Equal(newContent, oldContent) { + log.Log.V(2).Info("WriteSwitchdevConfFile(): no update") + return false, nil + } + log.Log.V(2).Info("WriteSwitchdevConfFile(): write to switchdev.conf", "content", newContent) + err = os.WriteFile(consts.SriovHostSwitchDevConfPath, newContent, 0644) + if err != nil { + log.Log.Error(err, "WriteSwitchdevConfFile(): failed to write file") + return false, err + } + return true, nil +} + +func (u *udev) AddUdevRule(pfPciAddress string) error { + log.Log.V(2).Info("AddUdevRule()", "device", pfPciAddress) + pathFile := filepath.Join(vars.FilesystemRoot, consts.UdevRulesFolder) + udevRuleContent := fmt.Sprintf(consts.NMUdevRule, strings.Join(vars.SupportedVfIds, "|"), pfPciAddress) + + err := os.MkdirAll(pathFile, os.ModePerm) + if err != nil && !os.IsExist(err) { + log.Log.Error(err, "AddUdevRule(): failed to create dir", "path", pathFile) + return err + } + + filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) + // if the file does not exist or if oldContent != newContent + // write to file and create it if it doesn't exist + err = os.WriteFile(filePath, []byte(udevRuleContent), 0666) + if err != nil { + log.Log.Error(err, "AddUdevRule(): fail to write file", "path", filePath) + return err + } + return nil +} + +func (u *udev) RemoveUdevRule(pfPciAddress string) error { + pathFile := filepath.Join(vars.FilesystemRoot, consts.UdevRulesFolder) + filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) + err := os.Remove(filePath) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} diff --git a/pkg/platforms/mock/mock_platforms.go b/pkg/platforms/mock/mock_platforms.go new file mode 100644 index 000000000..40eb5dc4c --- /dev/null +++ b/pkg/platforms/mock/mock_platforms.go @@ -0,0 +1,134 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: platforms.go + +// Package mock_platforms is a generated GoMock package. +package mock_platforms + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + openshift "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" + versioned "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" +) + +// MockInterface is a mock of Interface interface. +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface. +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance. +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// CreateOpenstackDevicesInfo mocks base method. +func (m *MockInterface) CreateOpenstackDevicesInfo() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOpenstackDevicesInfo") + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOpenstackDevicesInfo indicates an expected call of CreateOpenstackDevicesInfo. +func (mr *MockInterfaceMockRecorder) CreateOpenstackDevicesInfo() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfo", reflect.TypeOf((*MockInterface)(nil).CreateOpenstackDevicesInfo)) +} + +// CreateOpenstackDevicesInfoFromNodeStatus mocks base method. +func (m *MockInterface) CreateOpenstackDevicesInfoFromNodeStatus(arg0 *v1.SriovNetworkNodeState) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "CreateOpenstackDevicesInfoFromNodeStatus", arg0) +} + +// CreateOpenstackDevicesInfoFromNodeStatus indicates an expected call of CreateOpenstackDevicesInfoFromNodeStatus. +func (mr *MockInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfoFromNodeStatus", reflect.TypeOf((*MockInterface)(nil).CreateOpenstackDevicesInfoFromNodeStatus), arg0) +} + +// DiscoverSriovDevicesVirtual mocks base method. +func (m *MockInterface) DiscoverSriovDevicesVirtual() ([]v1.InterfaceExt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverSriovDevicesVirtual") + ret0, _ := ret[0].([]v1.InterfaceExt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiscoverSriovDevicesVirtual indicates an expected call of DiscoverSriovDevicesVirtual. +func (mr *MockInterfaceMockRecorder) DiscoverSriovDevicesVirtual() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevicesVirtual", reflect.TypeOf((*MockInterface)(nil).DiscoverSriovDevicesVirtual)) +} + +// GetFlavor mocks base method. +func (m *MockInterface) GetFlavor() openshift.OpenshiftFlavor { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFlavor") + ret0, _ := ret[0].(openshift.OpenshiftFlavor) + return ret0 +} + +// GetFlavor indicates an expected call of GetFlavor. +func (mr *MockInterfaceMockRecorder) GetFlavor() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFlavor", reflect.TypeOf((*MockInterface)(nil).GetFlavor)) +} + +// GetMcClient mocks base method. +func (m *MockInterface) GetMcClient() versioned.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMcClient") + ret0, _ := ret[0].(versioned.Interface) + return ret0 +} + +// GetMcClient indicates an expected call of GetMcClient. +func (mr *MockInterfaceMockRecorder) GetMcClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMcClient", reflect.TypeOf((*MockInterface)(nil).GetMcClient)) +} + +// IsHypershift mocks base method. +func (m *MockInterface) IsHypershift() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsHypershift") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsHypershift indicates an expected call of IsHypershift. +func (mr *MockInterfaceMockRecorder) IsHypershift() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsHypershift", reflect.TypeOf((*MockInterface)(nil).IsHypershift)) +} + +// IsOpenshiftCluster mocks base method. +func (m *MockInterface) IsOpenshiftCluster() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsOpenshiftCluster") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsOpenshiftCluster indicates an expected call of IsOpenshiftCluster. +func (mr *MockInterfaceMockRecorder) IsOpenshiftCluster() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsOpenshiftCluster", reflect.TypeOf((*MockInterface)(nil).IsOpenshiftCluster)) +} diff --git a/pkg/platforms/openshift/mock/mock_openshift.go b/pkg/platforms/openshift/mock/mock_openshift.go new file mode 100644 index 000000000..fb307a36e --- /dev/null +++ b/pkg/platforms/openshift/mock/mock_openshift.go @@ -0,0 +1,92 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: openshift.go + +// Package mock_openshift is a generated GoMock package. +package mock_openshift + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + openshift "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" + versioned "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" +) + +// MockOpenshiftContextInterface is a mock of OpenshiftContextInterface interface. +type MockOpenshiftContextInterface struct { + ctrl *gomock.Controller + recorder *MockOpenshiftContextInterfaceMockRecorder +} + +// MockOpenshiftContextInterfaceMockRecorder is the mock recorder for MockOpenshiftContextInterface. +type MockOpenshiftContextInterfaceMockRecorder struct { + mock *MockOpenshiftContextInterface +} + +// NewMockOpenshiftContextInterface creates a new mock instance. +func NewMockOpenshiftContextInterface(ctrl *gomock.Controller) *MockOpenshiftContextInterface { + mock := &MockOpenshiftContextInterface{ctrl: ctrl} + mock.recorder = &MockOpenshiftContextInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockOpenshiftContextInterface) EXPECT() *MockOpenshiftContextInterfaceMockRecorder { + return m.recorder +} + +// GetFlavor mocks base method. +func (m *MockOpenshiftContextInterface) GetFlavor() openshift.OpenshiftFlavor { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFlavor") + ret0, _ := ret[0].(openshift.OpenshiftFlavor) + return ret0 +} + +// GetFlavor indicates an expected call of GetFlavor. +func (mr *MockOpenshiftContextInterfaceMockRecorder) GetFlavor() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFlavor", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).GetFlavor)) +} + +// GetMcClient mocks base method. +func (m *MockOpenshiftContextInterface) GetMcClient() versioned.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMcClient") + ret0, _ := ret[0].(versioned.Interface) + return ret0 +} + +// GetMcClient indicates an expected call of GetMcClient. +func (mr *MockOpenshiftContextInterfaceMockRecorder) GetMcClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMcClient", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).GetMcClient)) +} + +// IsHypershift mocks base method. +func (m *MockOpenshiftContextInterface) IsHypershift() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsHypershift") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsHypershift indicates an expected call of IsHypershift. +func (mr *MockOpenshiftContextInterfaceMockRecorder) IsHypershift() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsHypershift", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).IsHypershift)) +} + +// IsOpenshiftCluster mocks base method. +func (m *MockOpenshiftContextInterface) IsOpenshiftCluster() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsOpenshiftCluster") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsOpenshiftCluster indicates an expected call of IsOpenshiftCluster. +func (mr *MockOpenshiftContextInterfaceMockRecorder) IsOpenshiftCluster() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsOpenshiftCluster", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).IsOpenshiftCluster)) +} diff --git a/pkg/utils/openshift_context.go b/pkg/platforms/openshift/openshift.go similarity index 50% rename from pkg/utils/openshift_context.go rename to pkg/platforms/openshift/openshift.go index 44a5b5e41..7ba2b6e29 100644 --- a/pkg/utils/openshift_context.go +++ b/pkg/platforms/openshift/openshift.go @@ -1,10 +1,12 @@ -package utils +package openshift import ( mcclientset "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) // OpenshiftFlavor holds metadata about the type of Openshift environment the operator is in. @@ -17,8 +19,16 @@ const ( OpenshiftFlavorDefault OpenshiftFlavor = "default" ) -// OpenshiftContext contains metadata and structs utilized to interact with Openshift clusters -type OpenshiftContext struct { +//go:generate ../../../bin/mockgen -destination mock/mock_openshift.go -source openshift.go +type OpenshiftContextInterface interface { + GetFlavor() OpenshiftFlavor + GetMcClient() mcclientset.Interface + IsOpenshiftCluster() bool + IsHypershift() bool +} + +// openshiftContext contains metadata and structs utilized to interact with Openshift clusters +type openshiftContext struct { // McClient is a client for MachineConfigs in an Openshift environment McClient mcclientset.Interface @@ -29,25 +39,25 @@ type OpenshiftContext struct { OpenshiftFlavor OpenshiftFlavor } -func NewOpenshiftContext(config *rest.Config, scheme *runtime.Scheme) (*OpenshiftContext, error) { - if ClusterType != ClusterTypeOpenshift { - return &OpenshiftContext{nil, false, ""}, nil +func New() (OpenshiftContextInterface, error) { + if vars.ClusterType != consts.ClusterTypeOpenshift { + return &openshiftContext{nil, false, ""}, nil } - mcclient, err := mcclientset.NewForConfig(config) + mcclient, err := mcclientset.NewForConfig(vars.Config) if err != nil { return nil, err } openshiftFlavor := OpenshiftFlavorDefault - infraClient, err := client.New(config, client.Options{ - Scheme: scheme, + infraClient, err := client.New(vars.Config, client.Options{ + Scheme: vars.Scheme, }) if err != nil { return nil, err } - isHypershift, err := IsExternalControlPlaneCluster(infraClient) + isHypershift, err := utils.IsExternalControlPlaneCluster(infraClient) if err != nil { return nil, err } @@ -56,13 +66,21 @@ func NewOpenshiftContext(config *rest.Config, scheme *runtime.Scheme) (*Openshif openshiftFlavor = OpenshiftFlavorHypershift } - return &OpenshiftContext{mcclient, true, openshiftFlavor}, nil + return &openshiftContext{mcclient, true, openshiftFlavor}, nil +} + +func (c *openshiftContext) GetFlavor() OpenshiftFlavor { + return c.OpenshiftFlavor +} + +func (c *openshiftContext) GetMcClient() mcclientset.Interface { + return c.McClient } -func (c OpenshiftContext) IsOpenshiftCluster() bool { +func (c openshiftContext) IsOpenshiftCluster() bool { return c.IsOpenShiftCluster } -func (c OpenshiftContext) IsHypershift() bool { +func (c openshiftContext) IsHypershift() bool { return c.OpenshiftFlavor == OpenshiftFlavorHypershift } diff --git a/pkg/platforms/openstack/mock/mock_openstack.go b/pkg/platforms/openstack/mock/mock_openstack.go new file mode 100644 index 000000000..9ef989297 --- /dev/null +++ b/pkg/platforms/openstack/mock/mock_openstack.go @@ -0,0 +1,76 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: openstack.go + +// Package mock_openstack is a generated GoMock package. +package mock_openstack + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +// MockOpenstackInterface is a mock of OpenstackInterface interface. +type MockOpenstackInterface struct { + ctrl *gomock.Controller + recorder *MockOpenstackInterfaceMockRecorder +} + +// MockOpenstackInterfaceMockRecorder is the mock recorder for MockOpenstackInterface. +type MockOpenstackInterfaceMockRecorder struct { + mock *MockOpenstackInterface +} + +// NewMockOpenstackInterface creates a new mock instance. +func NewMockOpenstackInterface(ctrl *gomock.Controller) *MockOpenstackInterface { + mock := &MockOpenstackInterface{ctrl: ctrl} + mock.recorder = &MockOpenstackInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockOpenstackInterface) EXPECT() *MockOpenstackInterfaceMockRecorder { + return m.recorder +} + +// CreateOpenstackDevicesInfo mocks base method. +func (m *MockOpenstackInterface) CreateOpenstackDevicesInfo() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOpenstackDevicesInfo") + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOpenstackDevicesInfo indicates an expected call of CreateOpenstackDevicesInfo. +func (mr *MockOpenstackInterfaceMockRecorder) CreateOpenstackDevicesInfo() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfo", reflect.TypeOf((*MockOpenstackInterface)(nil).CreateOpenstackDevicesInfo)) +} + +// CreateOpenstackDevicesInfoFromNodeStatus mocks base method. +func (m *MockOpenstackInterface) CreateOpenstackDevicesInfoFromNodeStatus(arg0 *v1.SriovNetworkNodeState) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "CreateOpenstackDevicesInfoFromNodeStatus", arg0) +} + +// CreateOpenstackDevicesInfoFromNodeStatus indicates an expected call of CreateOpenstackDevicesInfoFromNodeStatus. +func (mr *MockOpenstackInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfoFromNodeStatus", reflect.TypeOf((*MockOpenstackInterface)(nil).CreateOpenstackDevicesInfoFromNodeStatus), arg0) +} + +// DiscoverSriovDevicesVirtual mocks base method. +func (m *MockOpenstackInterface) DiscoverSriovDevicesVirtual() ([]v1.InterfaceExt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverSriovDevicesVirtual") + ret0, _ := ret[0].([]v1.InterfaceExt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiscoverSriovDevicesVirtual indicates an expected call of DiscoverSriovDevicesVirtual. +func (mr *MockOpenstackInterfaceMockRecorder) DiscoverSriovDevicesVirtual() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevicesVirtual", reflect.TypeOf((*MockOpenstackInterface)(nil).DiscoverSriovDevicesVirtual)) +} diff --git a/pkg/utils/utils_virtual.go b/pkg/platforms/openstack/openstack.go similarity index 67% rename from pkg/utils/utils_virtual.go rename to pkg/platforms/openstack/openstack.go index a5e6ddc84..658995727 100644 --- a/pkg/utils/utils_virtual.go +++ b/pkg/platforms/openstack/openstack.go @@ -1,12 +1,10 @@ -package utils +package openstack import ( "encoding/json" - "errors" "fmt" "io" "os" - "path/filepath" "strconv" "strings" @@ -18,35 +16,8 @@ import ( dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" -) - -// PlatformType ... -type PlatformType int - -const ( - // Baremetal platform - Baremetal PlatformType = iota - // VirtualOpenStack ... - VirtualOpenStack -) - -func (e PlatformType) String() string { - switch e { - case Baremetal: - return "Baremetal" - case VirtualOpenStack: - return "Virtual/Openstack" - default: - return fmt.Sprintf("%d", int(e)) - } -} - -var ( - // PlatformMap contains supported platforms for virtual VF - PlatformMap = map[string]PlatformType{ - "openstack": VirtualOpenStack, - } + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" ) const ( @@ -64,6 +35,18 @@ var ( ospMetaDataFile = ospMetaDataDir + "/meta_data.json" ) +//go:generate ../../../bin/mockgen -destination mock/mock_openstack.go -source openstack.go +type OpenstackInterface interface { + CreateOpenstackDevicesInfo() error + CreateOpenstackDevicesInfoFromNodeStatus(*sriovnetworkv1.SriovNetworkNodeState) + DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error) +} + +type openstackContext struct { + hostManager host.HostManagerInterface + openStackDevicesInfo OSPDevicesInfo +} + // OSPMetaDataDevice -- Device structure within meta_data.json type OSPMetaDataDevice struct { Vlan int `json:"vlan,omitempty"` @@ -117,8 +100,12 @@ type OSPDeviceInfo struct { NetworkID string } +func New() OpenstackInterface { + return &openstackContext{} +} + // GetOpenstackData gets the metadata and network_data -func GetOpenstackData(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { +func getOpenstackData(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { metaData, networkData, err = getOpenstackDataFromConfigDrive(useHostPath) if err != nil { metaData, networkData, err = getOpenstackDataFromMetadataService() @@ -267,11 +254,19 @@ func getPCIAddressFromMACAddress(macAddress string, nics []*net.NIC) (string, er } // CreateOpenstackDevicesInfo create the openstack device info map -func CreateOpenstackDevicesInfo(metaData *OSPMetaData, networkData *OSPNetworkData) (OSPDevicesInfo, error) { +func (o *openstackContext) CreateOpenstackDevicesInfo() error { log.Log.Info("CreateOpenstackDevicesInfo()") devicesInfo := make(OSPDevicesInfo) + + metaData, networkData, err := getOpenstackData(true) + if err != nil { + log.Log.Error(err, "failed to read OpenStack data") + return err + } + if metaData == nil || networkData == nil { - return nil, nil + o.openStackDevicesInfo = make(OSPDevicesInfo) + return nil } // use this for hw pass throw interfaces @@ -291,12 +286,12 @@ func CreateOpenstackDevicesInfo(metaData *OSPMetaData, networkData *OSPNetworkDa // for vhostuser interface type we check the interfaces on the node pci, err := ghw.PCI() if err != nil { - return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err) + return fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err) } devices := pci.ListDevices() if len(devices) == 0 { - return nil, fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices") + return fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices") } for _, device := range devices { @@ -311,14 +306,14 @@ func CreateOpenstackDevicesInfo(metaData *OSPMetaData, networkData *OSPNetworkDa "device", device) continue } - if devClass != netClass { + if devClass != consts.NetClass { // Not network device continue } macAddress := "" - if name := tryToGetVirtualInterfaceName(device.Address); name != "" { - if mac := getNetDevMac(name); mac != "" { + if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" { + if mac := o.hostManager.GetNetDevMac(name); mac != "" { macAddress = mac } } @@ -339,11 +334,12 @@ func CreateOpenstackDevicesInfo(metaData *OSPMetaData, networkData *OSPNetworkDa } } - return devicesInfo, err + o.openStackDevicesInfo = devicesInfo + return nil } // DiscoverSriovDevicesVirtual discovers VFs on a virtual platform -func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.InterfaceExt, error) { +func (o *openstackContext) DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.InterfaceExt, error) { log.Log.V(2).Info("DiscoverSriovDevicesVirtual()") pfList := []sriovnetworkv1.InterfaceExt{} @@ -364,12 +360,12 @@ func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.I "device", device) continue } - if devClass != netClass { + if devClass != consts.NetClass { // Not network device continue } - deviceInfo, exist := devicesInfo[device.Address] + deviceInfo, exist := o.openStackDevicesInfo[device.Address] if !exist { log.Log.Error(nil, "DiscoverSriovDevicesVirtual(): unable to find device in devicesInfo list, skipping", "device", device.Address) @@ -391,17 +387,17 @@ func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.I DeviceID: device.Product.ID, NetFilter: netFilter, } - if mtu := getNetdevMTU(device.Address); mtu > 0 { + if mtu := o.hostManager.GetNetdevMTU(device.Address); mtu > 0 { iface.Mtu = mtu } - if name := tryToGetVirtualInterfaceName(device.Address); name != "" { + if name := o.hostManager.TryToGetVirtualInterfaceName(device.Address); name != "" { iface.Name = name - if iface.Mac = getNetDevMac(name); iface.Mac == "" { + if iface.Mac = o.hostManager.GetNetDevMac(name); iface.Mac == "" { iface.Mac = metaMac } - iface.LinkSpeed = getNetDevLinkSpeed(name) + iface.LinkSpeed = o.hostManager.GetNetDevLinkSpeed(name) } - iface.LinkType = getLinkType(iface) + iface.LinkType = o.hostManager.GetLinkType(iface) iface.TotalVfs = 1 iface.NumVfs = 1 @@ -422,147 +418,11 @@ func DiscoverSriovDevicesVirtual(devicesInfo OSPDevicesInfo) ([]sriovnetworkv1.I return pfList, nil } -func CreateOpenstackDevicesInfoFromNodeStatus(networkState *sriovnetworkv1.SriovNetworkNodeState) OSPDevicesInfo { +func (o *openstackContext) CreateOpenstackDevicesInfoFromNodeStatus(networkState *sriovnetworkv1.SriovNetworkNodeState) { devicesInfo := make(OSPDevicesInfo) for _, iface := range networkState.Status.Interfaces { devicesInfo[iface.PciAddress] = &OSPDeviceInfo{MacAddress: iface.Mac, NetworkID: iface.NetFilter} } - return devicesInfo -} - -// tryToGetVirtualInterfaceName get the interface name of a virtio interface -func tryToGetVirtualInterfaceName(pciAddr string) string { - log.Log.Info("tryToGetVirtualInterfaceName() get interface name for device", "device", pciAddr) - - // To support different driver that is not virtio-pci like mlx - name := tryGetInterfaceName(pciAddr) - if name != "" { - return name - } - - netDir, err := filepath.Glob(filepath.Join(sysBusPciDevices, pciAddr, "virtio*", "net")) - if err != nil || len(netDir) < 1 { - return "" - } - - fInfos, err := os.ReadDir(netDir[0]) - if err != nil { - log.Log.Error(err, "tryToGetVirtualInterfaceName(): failed to read net directory", "dir", netDir[0]) - return "" - } - - names := make([]string, 0) - for _, f := range fInfos { - names = append(names, f.Name()) - } - - if len(names) < 1 { - return "" - } - - return names[0] -} - -// SyncNodeStateVirtual attempt to update the node state to match the desired state -// -// in virtual platforms -func SyncNodeStateVirtual(newState *sriovnetworkv1.SriovNetworkNodeState) error { - var err error - for _, ifaceStatus := range newState.Status.Interfaces { - for _, iface := range newState.Spec.Interfaces { - if iface.PciAddress == ifaceStatus.PciAddress { - if !needUpdateVirtual(&iface, &ifaceStatus) { - log.Log.V(2).Info("SyncNodeStateVirtual(): no need update interface", "address", iface.PciAddress) - break - } - if err = configSriovDeviceVirtual(&iface, &ifaceStatus); err != nil { - log.Log.Error(err, "SyncNodeStateVirtual(): fail to config sriov interface", "address", iface.PciAddress) - return err - } - break - } - } - } - return nil -} - -func needUpdateVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) bool { - // The device MTU is set by the platorm - // The NumVfs is always 1 - if iface.NumVfs > 0 { - for _, vf := range ifaceStatus.VFs { - ingroup := false - for _, group := range iface.VfGroups { - if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) { - ingroup = true - if group.DeviceType != constants.DeviceTypeNetDevice { - if group.DeviceType != vf.Driver { - log.Log.V(2).Info("needUpdateVirtual(): Driver needs update", - "desired", group.DeviceType, "current", vf.Driver) - return true - } - } else { - if sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) { - log.Log.V(2).Info("needUpdateVirtual(): Driver needs update", - "desired", group.DeviceType, "current", vf.Driver) - return true - } - } - break - } - } - if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) { - // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver. - return true - } - } - } - return false -} - -func configSriovDeviceVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) error { - log.Log.V(2).Info("configSriovDeviceVirtual(): config interface", "address", iface.PciAddress, "config", iface) - // Config VFs - if iface.NumVfs > 0 { - if iface.NumVfs > 1 { - log.Log.Error(nil, "configSriovDeviceVirtual(): in a virtual environment, only one VF per interface", - "numVfs", iface.NumVfs) - return errors.New("NumVfs > 1") - } - if len(iface.VfGroups) != 1 { - log.Log.Error(nil, "configSriovDeviceVirtual(): missing VFGroup") - return errors.New("NumVfs != 1") - } - addr := iface.PciAddress - log.Log.V(2).Info("configSriovDeviceVirtual()", "address", addr) - driver := "" - vfID := 0 - for _, group := range iface.VfGroups { - log.Log.V(2).Info("configSriovDeviceVirtual()", "group", group) - if sriovnetworkv1.IndexInRange(vfID, group.VfRange) { - log.Log.V(2).Info("configSriovDeviceVirtual()", "indexInRange", vfID) - if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) { - log.Log.V(2).Info("configSriovDeviceVirtual()", "driver", group.DeviceType) - driver = group.DeviceType - } - break - } - } - if driver == "" { - log.Log.V(2).Info("configSriovDeviceVirtual(): bind default") - if err := BindDefaultDriver(addr); err != nil { - log.Log.Error(err, "configSriovDeviceVirtual(): fail to bind default driver", "device", addr) - return err - } - } else { - log.Log.V(2).Info("configSriovDeviceVirtual(): bind driver", "driver", driver) - if err := BindDpdkDriver(addr, driver); err != nil { - log.Log.Error(err, "configSriovDeviceVirtual(): fail to bind driver for device", - "driver", driver, "device", addr) - return err - } - } - } - return nil + o.openStackDevicesInfo = devicesInfo } diff --git a/pkg/utils/utils_virtual_test.go b/pkg/platforms/openstack/openstack_test.go similarity index 95% rename from pkg/utils/utils_virtual_test.go rename to pkg/platforms/openstack/openstack_test.go index 5c55d4b54..ca18bce6e 100644 --- a/pkg/utils/utils_virtual_test.go +++ b/pkg/platforms/openstack/openstack_test.go @@ -1,4 +1,4 @@ -package utils +package openstack import ( "testing" @@ -6,10 +6,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/utils/pointer" + "github.com/jaypipes/ghw" "github.com/jaypipes/ghw/pkg/net" "github.com/jaypipes/ghw/pkg/option" - "k8s.io/utils/pointer" ) func TestUtilsVirtual(t *testing.T) { @@ -44,7 +45,7 @@ var _ = Describe("Virtual", func() { ghw.Network = net.New }) - metaData, _, err := GetOpenstackData(false) + metaData, _, err := getOpenstackData(false) Expect(err).ToNot(HaveOccurred()) Expect(metaData.Devices).To(HaveLen(2)) diff --git a/pkg/utils/testdata/meta_data.json b/pkg/platforms/openstack/testdata/meta_data.json similarity index 100% rename from pkg/utils/testdata/meta_data.json rename to pkg/platforms/openstack/testdata/meta_data.json diff --git a/pkg/utils/testdata/network_data.json b/pkg/platforms/openstack/testdata/network_data.json similarity index 100% rename from pkg/utils/testdata/network_data.json rename to pkg/platforms/openstack/testdata/network_data.json diff --git a/pkg/platforms/platforms.go b/pkg/platforms/platforms.go new file mode 100644 index 000000000..529f51821 --- /dev/null +++ b/pkg/platforms/platforms.go @@ -0,0 +1,31 @@ +package platforms + +import ( + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openstack" +) + +//go:generate ../../bin/mockgen -destination mock/mock_platforms.go -source platforms.go +type Interface interface { + openshift.OpenshiftContextInterface + openstack.OpenstackInterface +} + +type platformHelper struct { + openshift.OpenshiftContextInterface + openstack.OpenstackInterface +} + +func NewDefaultPlatformHelper() (Interface, error) { + openshiftContext, err := openshift.New() + if err != nil { + return nil, err + } + + openstackContext := openstack.New() + + return &platformHelper{ + openshiftContext, + openstackContext, + }, nil +} diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 44069407e..88aa38549 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -3,6 +3,7 @@ package generic import ( "bytes" "errors" + "fmt" "os/exec" "reflect" "strconv" @@ -12,10 +13,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" ) var PluginName = "generic_plugin" @@ -54,34 +57,33 @@ type GenericPlugin struct { LastState *sriovnetworkv1.SriovNetworkNodeState DriverStateMap DriverStateMapType DesiredKernelArgs map[string]bool - RunningOnHost bool - HostManager host.HostManagerInterface - StoreManager utils.StoreManagerInterface + pfsToSkip map[string]bool + helpers helper.HostHelpersInterface } const scriptsPath = "bindata/scripts/enable-kargs.sh" // Initialize our plugin and set up initial values -func NewGenericPlugin(runningOnHost bool, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface) (plugin.VendorPlugin, error) { +func NewGenericPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) { driverStateMap := make(map[uint]*DriverState) driverStateMap[Vfio] = &DriverState{ DriverName: vfioPciDriver, - DeviceType: constants.DeviceTypeVfioPci, + DeviceType: consts.DeviceTypeVfioPci, VdpaType: "", NeedDriverFunc: needDriverCheckDeviceType, DriverLoaded: false, } driverStateMap[VirtioVdpa] = &DriverState{ DriverName: virtioVdpaDriver, - DeviceType: constants.DeviceTypeNetDevice, - VdpaType: constants.VdpaTypeVirtio, + DeviceType: consts.DeviceTypeNetDevice, + VdpaType: consts.VdpaTypeVirtio, NeedDriverFunc: needDriverCheckVdpaType, DriverLoaded: false, } driverStateMap[VhostVdpa] = &DriverState{ DriverName: vhostVdpaDriver, - DeviceType: constants.DeviceTypeNetDevice, - VdpaType: constants.VdpaTypeVhost, + DeviceType: consts.DeviceTypeNetDevice, + VdpaType: consts.VdpaTypeVhost, NeedDriverFunc: needDriverCheckVdpaType, DriverLoaded: false, } @@ -90,9 +92,8 @@ func NewGenericPlugin(runningOnHost bool, hostManager host.HostManagerInterface, SpecVersion: "1.0", DriverStateMap: driverStateMap, DesiredKernelArgs: make(map[string]bool), - RunningOnHost: runningOnHost, - HostManager: hostManager, - StoreManager: storeManager, + pfsToSkip: make(map[string]bool), + helpers: helpers, }, nil } @@ -109,9 +110,6 @@ func (p *GenericPlugin) Spec() string { // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { log.Log.Info("generic-plugin OnNodeStateChange()") - needDrain = false - needReboot = false - err = nil p.DesireState = new needDrain = p.needDrainNode(new.Spec.Interfaces, new.Status.Interfaces) @@ -130,7 +128,7 @@ func (p *GenericPlugin) syncDriverState() error { for _, driverState := range p.DriverStateMap { if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) { log.Log.V(2).Info("loading driver", "name", driverState.DriverName) - if err := p.HostManager.LoadKernelModule(driverState.DriverName); err != nil { + if err := p.helpers.LoadKernelModule(driverState.DriverName); err != nil { log.Log.Error(err, "generic-plugin syncDriverState(): fail to load kmod", "name", driverState.DriverName) return err } @@ -156,27 +154,19 @@ func (p *GenericPlugin) Apply() error { return err } - // Create a map with all the PFs we will need to configure - // we need to create it here before we access the host file system using the chroot function - // because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system - pfsToSkip, err := utils.GetPfsToSkip(p.DesireState) - if err != nil { - return err - } - // When calling from systemd do not try to chroot - if !p.RunningOnHost { - exit, err := utils.Chroot("/host") + if !vars.UsingSystemdMode { + exit, err := p.helpers.Chroot(consts.Host) if err != nil { return err } defer exit() } - if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil { + if err := p.helpers.ConfigSriovInterfaces(p.helpers, p.DesireState.Spec.Interfaces, p.DesireState.Status.Interfaces, p.pfsToSkip); err != nil { // Catch the "cannot allocate memory" error and try to use PCI realloc if errors.Is(err, syscall.ENOMEM) { - p.addToDesiredKernelArgs(utils.KernelArgPciRealloc) + p.addToDesiredKernelArgs(consts.KernelArgPciRealloc) } return err } @@ -217,7 +207,7 @@ func setKernelArg(karg string) (bool, error) { if err := cmd.Run(); err != nil { // if grubby is not there log and assume kernel args are set correctly. - if isCommandNotFound(err) { + if utils.IsCommandNotFound(err) { log.Log.Error(err, "generic-plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set", "kargs", karg) return false, nil @@ -236,15 +226,6 @@ func setKernelArg(karg string) (bool, error) { return false, err } -func isCommandNotFound(err error) bool { - if exitErr, ok := err.(*exec.ExitError); ok { - if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 { - return true - } - } - return false -} - // addToDesiredKernelArgs Should be called to queue a kernel arg to be added to the node. func (p *GenericPlugin) addToDesiredKernelArgs(karg string) { if _, ok := p.DesiredKernelArgs[karg]; !ok { @@ -259,12 +240,12 @@ func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) { if len(p.DesiredKernelArgs) == 0 { return false, nil } - kargs, err := utils.GetCurrentKernelArgs(false) + kargs, err := p.helpers.GetCurrentKernelArgs() if err != nil { return false, err } for desiredKarg, attempted := range p.DesiredKernelArgs { - set := utils.IsKernelArgsSet(kargs, desiredKarg) + set := p.helpers.IsKernelArgsSet(kargs, desiredKarg) if !set { if attempted { log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg", @@ -302,7 +283,7 @@ func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.Interfaces, current "address", iface.PciAddress) break } - if utils.NeedUpdate(&iface, &ifaceStatus) { + if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) { log.Log.V(2).Info("generic-plugin needDrainNode(): need drain, for PCI address request update", "address", iface.PciAddress) needDrain = true @@ -314,7 +295,7 @@ func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.Interfaces, current } if !configured && ifaceStatus.NumVfs > 0 { // load the PF info - pfStatus, exist, err := p.StoreManager.LoadPfsStatus(ifaceStatus.PciAddress) + pfStatus, exist, err := p.helpers.LoadPfsStatus(ifaceStatus.PciAddress) if err != nil { log.Log.Error(err, "generic-plugin needDrainNode(): failed to load info about PF status for pci device", "address", ifaceStatus.PciAddress) @@ -347,8 +328,8 @@ func (p *GenericPlugin) needDrainNode(desired sriovnetworkv1.Interfaces, current func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) { driverState := p.DriverStateMap[Vfio] if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) { - p.addToDesiredKernelArgs(utils.KernelArgIntelIommu) - p.addToDesiredKernelArgs(utils.KernelArgIommuPt) + p.addToDesiredKernelArgs(consts.KernelArgIntelIommu) + p.addToDesiredKernelArgs(consts.KernelArgIommuPt) } } @@ -366,7 +347,16 @@ func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeSta needReboot = true } - updateNode, err = utils.WriteSwitchdevConfFile(state) + // Create a map with all the PFs we will need to configure + // we need to create it here before we access the host file system using the chroot function + // because the skipConfigVf needs the mstconfig package that exist only inside the sriov-config-daemon file system + pfsToSkip, err := getPfsToSkip(p.DesireState, p.helpers) + if err != nil { + return false, err + } + p.pfsToSkip = pfsToSkip + + updateNode, err = p.helpers.WriteSwitchdevConfFile(state, p.pfsToSkip) if err != nil { log.Log.Error(err, "generic-plugin needRebootNode(): fail to write switchdev device config file") return false, err @@ -379,6 +369,53 @@ func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeSta return needReboot, nil } +// getPfsToSkip return a map of devices pci addresses to should be configured via systemd instead if the legacy mode +// we skip devices in switchdev mode and Bluefield card in ConnectX mode +func getPfsToSkip(ns *sriovnetworkv1.SriovNetworkNodeState, mlxHelper mlx.MellanoxInterface) (map[string]bool, error) { + pfsToSkip := map[string]bool{} + for _, ifaceStatus := range ns.Status.Interfaces { + for _, iface := range ns.Spec.Interfaces { + if iface.PciAddress == ifaceStatus.PciAddress { + skip, err := skipConfigVf(iface, ifaceStatus, mlxHelper) + if err != nil { + log.Log.Error(err, "GetPfsToSkip(): fail to check for skip VFs", "device", iface.PciAddress) + return pfsToSkip, err + } + pfsToSkip[iface.PciAddress] = skip + break + } + } + } + + return pfsToSkip, nil +} + +// skipConfigVf Use systemd service to configure switchdev mode or BF-2 NICs in OpenShift +func skipConfigVf(ifSpec sriovnetworkv1.Interface, ifStatus sriovnetworkv1.InterfaceExt, mlxHelper mlx.MellanoxInterface) (bool, error) { + if ifSpec.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { + log.Log.V(2).Info("skipConfigVf(): skip config VF for switchdev device") + return true, nil + } + + // NVIDIA BlueField 2 and BlueField3 in OpenShift + if vars.ClusterType == consts.ClusterTypeOpenshift && ifStatus.Vendor == mlx.VendorMellanox && (ifStatus.DeviceID == mlx.DeviceBF2 || ifStatus.DeviceID == mlx.DeviceBF3) { + // TODO: remove this when switch to the systemd configuration support. + mode, err := mlxHelper.GetMellanoxBlueFieldMode(ifStatus.PciAddress) + if err != nil { + return false, fmt.Errorf("failed to read Mellanox Bluefield card mode for %s,%v", ifStatus.PciAddress, err) + } + + if mode == mlx.BluefieldConnectXMode { + return false, nil + } + + log.Log.V(2).Info("skipConfigVf(): skip config VF for Bluefiled card on DPU mode") + return true, nil + } + + return false, nil +} + // ////////////// for testing purposes only /////////////////////// func (p *GenericPlugin) getDriverStateMap() DriverStateMapType { return p.DriverStateMap diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index e1211b392..881fc87d4 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -8,9 +8,8 @@ import ( . "github.com/onsi/gomega" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - mock_host "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - mock_utils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils/mock" ) func TestGenericPlugin(t *testing.T) { @@ -24,16 +23,16 @@ var _ = Describe("Generic plugin", func() { genericPlugin plugin.VendorPlugin err error ctrl *gomock.Controller - mockHost *mock_host.MockHostManagerInterface - mockStore *mock_utils.MockStoreManagerInterface + hostHelper *mock_helper.MockHostHelpersInterface ) BeforeEach(func() { t = GinkgoT() ctrl = gomock.NewController(t) - mockHost = mock_host.NewMockHostManagerInterface(ctrl) - mockStore = mock_utils.NewMockStoreManagerInterface(ctrl) - genericPlugin, err = NewGenericPlugin(false, mockHost, mockStore) + + hostHelper = mock_helper.NewMockHostHelpersInterface(ctrl) + + genericPlugin, err = NewGenericPlugin(hostHelper) Expect(err).ToNot(HaveOccurred()) }) @@ -79,6 +78,7 @@ var _ = Describe("Generic plugin", func() { }, } + hostHelper.EXPECT().WriteSwitchdevConfFile(networkNodeState, map[string]bool{"0000:00:00.0": false}).Return(false, nil) needDrain, needReboot, err := genericPlugin.OnNodeStateChange(networkNodeState) Expect(err).ToNot(HaveOccurred()) Expect(needReboot).To(BeFalse()) @@ -134,6 +134,7 @@ var _ = Describe("Generic plugin", func() { }, } + hostHelper.EXPECT().WriteSwitchdevConfFile(networkNodeState, map[string]bool{"0000:00:00.0": false}).Return(false, nil) needDrain, needReboot, err := genericPlugin.OnNodeStateChange(networkNodeState) Expect(err).ToNot(HaveOccurred()) Expect(needReboot).To(BeFalse()) diff --git a/pkg/plugins/intel/intel_plugin.go b/pkg/plugins/intel/intel_plugin.go index bf032652e..1c64a47fb 100644 --- a/pkg/plugins/intel/intel_plugin.go +++ b/pkg/plugins/intel/intel_plugin.go @@ -4,6 +4,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" ) @@ -16,7 +17,7 @@ type IntelPlugin struct { LastState *sriovnetworkv1.SriovNetworkNodeState } -func NewIntelPlugin() (plugin.VendorPlugin, error) { +func NewIntelPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) { return &IntelPlugin{ PluginName: PluginName, SpecVersion: "1.0", diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index 4cbcc818e..1bf70eab1 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -6,13 +6,14 @@ import ( "path" "strings" - "github.com/coreos/go-systemd/v22/unit" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" plugins "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) var PluginName = "k8s_plugin" @@ -20,17 +21,16 @@ var PluginName = "k8s_plugin" type K8sPlugin struct { PluginName string SpecVersion string - serviceManager service.ServiceManager - switchdevBeforeNMRunScript *service.ScriptManifestFile - switchdevAfterNMRunScript *service.ScriptManifestFile - switchdevUdevScript *service.ScriptManifestFile - switchdevBeforeNMService *service.Service - switchdevAfterNMService *service.Service - openVSwitchService *service.Service - networkManagerService *service.Service - sriovService *service.Service + switchdevBeforeNMRunScript *host.ScriptManifestFile + switchdevAfterNMRunScript *host.ScriptManifestFile + switchdevUdevScript *host.ScriptManifestFile + switchdevBeforeNMService *host.Service + switchdevAfterNMService *host.Service + openVSwitchService *host.Service + networkManagerService *host.Service + sriovService *host.Service updateTarget *k8sUpdateTarget - useSystemdService bool + hostHelper helper.HostHelpersInterface } type k8sUpdateTarget struct { @@ -40,7 +40,7 @@ type k8sUpdateTarget struct { switchdevAfterNMRunScript bool switchdevUdevScript bool sriovScript bool - systemServices []*service.Service + systemServices []*host.Service } func (u *k8sUpdateTarget) needUpdate() bool { @@ -58,7 +58,7 @@ func (u *k8sUpdateTarget) reset() { u.switchdevAfterNMRunScript = false u.switchdevUdevScript = false u.sriovScript = false - u.systemServices = []*service.Service{} + u.systemServices = []*host.Service{} } func (u *k8sUpdateTarget) String() string { @@ -92,18 +92,15 @@ const ( configuresSwitchdevBeforeNMScript = switchdevManifestPath + "files/switchdev-configuration-before-nm.sh.yaml" configuresSwitchdevAfterNMScript = switchdevManifestPath + "files/switchdev-configuration-after-nm.sh.yaml" switchdevRenamingUdevScript = switchdevManifestPath + "files/switchdev-vf-link-name.sh.yaml" - - chroot = "/host" ) // Initialize our plugin and set up initial values -func NewK8sPlugin(useSystemdService bool) (plugins.VendorPlugin, error) { +func NewK8sPlugin(helper helper.HostHelpersInterface) (plugins.VendorPlugin, error) { k8sPluging := &K8sPlugin{ - PluginName: PluginName, - SpecVersion: "1.0", - serviceManager: service.NewServiceManager(chroot), - updateTarget: &k8sUpdateTarget{}, - useSystemdService: useSystemdService, + PluginName: PluginName, + SpecVersion: "1.0", + hostHelper: helper, + updateTarget: &k8sUpdateTarget{}, } return k8sPluging, k8sPluging.readManifestFiles() @@ -128,11 +125,11 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) p.updateTarget.reset() // TODO add check for enableOvsOffload in OperatorConfig later // Update services if switchdev required - if !p.useSystemdService && !utils.IsSwitchdevModeSpec(new.Spec) { + if !vars.UsingSystemdMode && !sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) { return } - if utils.IsSwitchdevModeSpec(new.Spec) { + if sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) { // Check services err = p.switchDevServicesStateUpdate() if err != nil { @@ -141,7 +138,7 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) } } - if p.useSystemdService { + if vars.UsingSystemdMode { // Check sriov service err = p.sriovServiceStateUpdate() if err != nil { @@ -170,14 +167,14 @@ func (p *K8sPlugin) Apply() error { return err } - if p.useSystemdService { + if vars.UsingSystemdMode { if err := p.updateSriovService(); err != nil { return err } } for _, systemService := range p.updateTarget.systemServices { - if err := p.updateSystemService(systemService); err != nil { + if err := p.hostHelper.UpdateSystemService(systemService); err != nil { return err } } @@ -187,26 +184,20 @@ func (p *K8sPlugin) Apply() error { func (p *K8sPlugin) readSwitchdevManifest() error { // Read switchdev service - switchdevBeforeNMService, err := service.ReadServiceManifestFile(switchdevBeforeNMUnitFile) + switchdevBeforeNMService, err := p.hostHelper.ReadServiceManifestFile(switchdevBeforeNMUnitFile) if err != nil { return err } - switchdevAfterNMService, err := service.ReadServiceManifestFile(switchdevAfterNMUnitFile) + switchdevAfterNMService, err := p.hostHelper.ReadServiceManifestFile(switchdevAfterNMUnitFile) if err != nil { return err } - // Remove run condition form the service - conditionOpt := &unit.UnitOption{ - Section: "Unit", - Name: "ConditionPathExists", - Value: "!/etc/ignition-machine-config-encapsulated.json", - } - switchdevBeforeNMService, err = service.RemoveFromService(switchdevBeforeNMService, conditionOpt) + switchdevBeforeNMService, err = p.hostHelper.RemoveFromService(switchdevBeforeNMService, host.ConditionOpt) if err != nil { return err } - switchdevAfterNMService, err = service.RemoveFromService(switchdevAfterNMService, conditionOpt) + switchdevAfterNMService, err = p.hostHelper.RemoveFromService(switchdevAfterNMService, host.ConditionOpt) if err != nil { return err } @@ -214,11 +205,11 @@ func (p *K8sPlugin) readSwitchdevManifest() error { p.switchdevAfterNMService = switchdevAfterNMService // Read switchdev run script - switchdevBeforeNMRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevBeforeNMScript) + switchdevBeforeNMRunScript, err := p.hostHelper.ReadScriptManifestFile(configuresSwitchdevBeforeNMScript) if err != nil { return err } - switchdevAfterNMRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevAfterNMScript) + switchdevAfterNMRunScript, err := p.hostHelper.ReadScriptManifestFile(configuresSwitchdevAfterNMScript) if err != nil { return err } @@ -226,7 +217,7 @@ func (p *K8sPlugin) readSwitchdevManifest() error { p.switchdevAfterNMRunScript = switchdevAfterNMRunScript // Read switchdev udev script - switchdevUdevScript, err := service.ReadScriptManifestFile(switchdevRenamingUdevScript) + switchdevUdevScript, err := p.hostHelper.ReadScriptManifestFile(switchdevRenamingUdevScript) if err != nil { return err } @@ -236,7 +227,7 @@ func (p *K8sPlugin) readSwitchdevManifest() error { } func (p *K8sPlugin) readNetworkManagerManifest() error { - networkManagerService, err := service.ReadServiceInjectionManifestFile(networkManagerUnitFile) + networkManagerService, err := p.hostHelper.ReadServiceInjectionManifestFile(networkManagerUnitFile) if err != nil { return err } @@ -246,7 +237,7 @@ func (p *K8sPlugin) readNetworkManagerManifest() error { } func (p *K8sPlugin) readOpenVSwitchdManifest() error { - openVSwitchService, err := service.ReadServiceInjectionManifestFile(ovsUnitFile) + openVSwitchService, err := p.hostHelper.ReadServiceInjectionManifestFile(ovsUnitFile) if err != nil { return err } @@ -256,7 +247,7 @@ func (p *K8sPlugin) readOpenVSwitchdManifest() error { } func (p *K8sPlugin) readSriovServiceManifest() error { - sriovService, err := service.ReadServiceManifestFile(sriovUnitFile) + sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovUnitFile) if err != nil { return err } @@ -322,7 +313,7 @@ func (p *K8sPlugin) switchdevServiceStateUpdate() error { func (p *K8sPlugin) sriovServiceStateUpdate() error { log.Log.Info("sriovServiceStateUpdate()") - isServiceEnabled, err := p.serviceManager.IsServiceEnabled(p.sriovService.Path) + isServiceEnabled, err := p.hostHelper.IsServiceEnabled(p.sriovService.Path) if err != nil { return err } @@ -340,12 +331,12 @@ func (p *K8sPlugin) sriovServiceStateUpdate() error { return nil } -func (p *K8sPlugin) getSwitchDevSystemServices() []*service.Service { - return []*service.Service{p.networkManagerService, p.openVSwitchService} +func (p *K8sPlugin) getSwitchDevSystemServices() []*host.Service { + return []*host.Service{p.networkManagerService, p.openVSwitchService} } -func (p *K8sPlugin) isSwitchdevScriptNeedUpdate(scriptObj *service.ScriptManifestFile) (needUpdate bool, err error) { - data, err := os.ReadFile(path.Join(chroot, scriptObj.Path)) +func (p *K8sPlugin) isSwitchdevScriptNeedUpdate(scriptObj *host.ScriptManifestFile) (needUpdate bool, err error) { + data, err := os.ReadFile(path.Join(consts.Host, scriptObj.Path)) if err != nil { if !os.IsNotExist(err) { return false, err @@ -357,8 +348,8 @@ func (p *K8sPlugin) isSwitchdevScriptNeedUpdate(scriptObj *service.ScriptManifes return false, nil } -func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *service.Service) (needUpdate bool, err error) { - swdService, err := p.serviceManager.ReadService(serviceObj.Path) +func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *host.Service) (needUpdate bool, err error) { + swdService, err := p.hostHelper.ReadService(serviceObj.Path) if err != nil { if !os.IsNotExist(err) { return false, err @@ -366,7 +357,7 @@ func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *service.Service) (n // service not exists return true, nil } else { - needChange, err := service.CompareServices(swdService, serviceObj) + needChange, err := p.hostHelper.CompareServices(swdService, serviceObj) if err != nil { return false, err } @@ -374,16 +365,16 @@ func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *service.Service) (n } } -func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool { +func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *host.Service) bool { log.Log.Info("isSystemServiceNeedUpdate()") - systemService, err := p.serviceManager.ReadService(serviceObj.Path) + systemService, err := p.hostHelper.ReadService(serviceObj.Path) if err != nil { log.Log.Error(err, "k8s-plugin isSystemServiceNeedUpdate(): failed to read sriov-config service file, ignoring", "path", serviceObj.Path) return false } if systemService != nil { - needChange, err := service.CompareServices(systemService, serviceObj) + needChange, err := p.hostHelper.CompareServices(systemService, serviceObj) if err != nil { log.Log.Error(err, "k8s-plugin isSystemServiceNeedUpdate(): failed to compare sriov-config service, ignoring") return false @@ -395,9 +386,9 @@ func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool } func (p *K8sPlugin) systemServicesStateUpdate() error { - var services []*service.Service + var services []*host.Service for _, systemService := range p.getSwitchDevSystemServices() { - exist, err := p.serviceManager.IsServiceExist(systemService.Path) + exist, err := p.hostHelper.IsServiceExist(systemService.Path) if err != nil { return err } @@ -431,7 +422,7 @@ func (p *K8sPlugin) switchDevServicesStateUpdate() error { func (p *K8sPlugin) updateSriovService() error { if p.updateTarget.sriovScript { - err := p.serviceManager.EnableService(p.sriovService) + err := p.hostHelper.EnableService(p.sriovService) if err != nil { return err } @@ -442,21 +433,21 @@ func (p *K8sPlugin) updateSriovService() error { func (p *K8sPlugin) updateSwitchdevService() error { if p.updateTarget.switchdevBeforeNMService { - err := p.serviceManager.EnableService(p.switchdevBeforeNMService) + err := p.hostHelper.EnableService(p.switchdevBeforeNMService) if err != nil { return err } } if p.updateTarget.switchdevAfterNMService { - err := p.serviceManager.EnableService(p.switchdevAfterNMService) + err := p.hostHelper.EnableService(p.switchdevAfterNMService) if err != nil { return err } } if p.updateTarget.switchdevBeforeNMRunScript { - err := os.WriteFile(path.Join(chroot, p.switchdevBeforeNMRunScript.Path), + err := os.WriteFile(path.Join(consts.Host, p.switchdevBeforeNMRunScript.Path), []byte(p.switchdevBeforeNMRunScript.Contents.Inline), 0755) if err != nil { return err @@ -464,7 +455,7 @@ func (p *K8sPlugin) updateSwitchdevService() error { } if p.updateTarget.switchdevAfterNMRunScript { - err := os.WriteFile(path.Join(chroot, p.switchdevAfterNMRunScript.Path), + err := os.WriteFile(path.Join(consts.Host, p.switchdevAfterNMRunScript.Path), []byte(p.switchdevAfterNMRunScript.Contents.Inline), 0755) if err != nil { return err @@ -472,7 +463,7 @@ func (p *K8sPlugin) updateSwitchdevService() error { } if p.updateTarget.switchdevUdevScript { - err := os.WriteFile(path.Join(chroot, p.switchdevUdevScript.Path), + err := os.WriteFile(path.Join(consts.Host, p.switchdevUdevScript.Path), []byte(p.switchdevUdevScript.Contents.Inline), 0755) if err != nil { return err @@ -481,32 +472,3 @@ func (p *K8sPlugin) updateSwitchdevService() error { return nil } - -func (p *K8sPlugin) updateSystemService(serviceObj *service.Service) error { - systemService, err := p.serviceManager.ReadService(serviceObj.Path) - if err != nil { - return err - } - if systemService == nil { - // Invalid case to reach here - return fmt.Errorf("k8s-plugin updateSystemService(): can't update non-existing service %q", serviceObj.Name) - } - serviceOptions, err := unit.Deserialize(strings.NewReader(serviceObj.Content)) - if err != nil { - return err - } - updatedService, err := service.AppendToService(systemService, serviceOptions...) - if err != nil { - return err - } - - return p.serviceManager.EnableService(updatedService) -} - -func (p *K8sPlugin) SetSystemdFlag() { - p.useSystemdService = true -} - -func (p *K8sPlugin) IsSystemService() bool { - return p.useSystemdService -} diff --git a/pkg/plugins/mellanox/mellanox_plugin.go b/pkg/plugins/mellanox/mellanox_plugin.go index 5f9f9f56a..d36996ca6 100644 --- a/pkg/plugins/mellanox/mellanox_plugin.go +++ b/pkg/plugins/mellanox/mellanox_plugin.go @@ -2,15 +2,13 @@ package mellanox import ( "fmt" - "strconv" - "strings" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" ) var PluginName = "mellanox_plugin" @@ -18,36 +16,21 @@ var PluginName = "mellanox_plugin" type MellanoxPlugin struct { PluginName string SpecVersion string + helpers helper.HostHelpersInterface } -type mlnxNic struct { - enableSriov bool - totalVfs int - linkTypeP1 string - linkTypeP2 string -} - -const ( - PreconfiguredLinkType = "Preconfigured" - UknownLinkType = "Uknown" - TotalVfs = "NUM_OF_VFS" - EnableSriov = "SRIOV_EN" - LinkTypeP1 = "LINK_TYPE_P1" - LinkTypeP2 = "LINK_TYPE_P2" - MellanoxVendorID = "15b3" -) - -var attributesToChange map[string]mlnxNic +var attributesToChange map[string]mlx.MlxNic var mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt var mellanoxNicsSpec map[string]sriovnetworkv1.Interface // Initialize our plugin and set up initial values -func NewMellanoxPlugin() (plugin.VendorPlugin, error) { +func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) { mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{} return &MellanoxPlugin{ PluginName: PluginName, SpecVersion: "1.0", + helpers: helpers, }, nil } @@ -68,18 +51,18 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS needDrain = false needReboot = false err = nil - attributesToChange = map[string]mlnxNic{} + attributesToChange = map[string]mlx.MlxNic{} mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{} processedNics := map[string]bool{} // Read mellanox NIC status once if len(mellanoxNicsStatus) == 0 { for _, iface := range new.Status.Interfaces { - if iface.Vendor != MellanoxVendorID { + if iface.Vendor != mlx.MellanoxVendorID { continue } - pciPrefix := getPciAddressPrefix(iface.PciAddress) + pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress) if ifaces, ok := mellanoxNicsStatus[pciPrefix]; ok { ifaces[iface.PciAddress] = iface } else { @@ -90,14 +73,14 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS // Add only mellanox cards that required changes in the map, to help track dual port NICs for _, iface := range new.Spec.Interfaces { - pciPrefix := getPciAddressPrefix(iface.PciAddress) + pciPrefix := mlx.GetPciAddressPrefix(iface.PciAddress) if _, ok := mellanoxNicsStatus[pciPrefix]; !ok { continue } mellanoxNicsSpec[iface.PciAddress] = iface } - if utils.IsKernelLockdownMode(false) { + if p.helpers.IsKernelLockdownMode() { if len(mellanoxNicsSpec) > 0 { log.Log.Info("Lockdown mode detected, failing on interface update for mellanox devices") return false, false, fmt.Errorf("mellanox device detected when in lockdown mode") @@ -107,25 +90,25 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS } for _, ifaceSpec := range mellanoxNicsSpec { - pciPrefix := getPciAddressPrefix(ifaceSpec.PciAddress) + pciPrefix := mlx.GetPciAddressPrefix(ifaceSpec.PciAddress) // skip processed nics, help not running the same logic 2 times for dual port NICs if _, ok := processedNics[pciPrefix]; ok { continue } processedNics[pciPrefix] = true - fwCurrent, fwNext, err := getMlnxNicFwData(ifaceSpec.PciAddress) + fwCurrent, fwNext, err := p.helpers.GetMlxNicFwData(ifaceSpec.PciAddress) if err != nil { return false, false, err } - isDualPort := isDualPort(ifaceSpec.PciAddress) + isDualPort := mlx.IsDualPort(ifaceSpec.PciAddress, mellanoxNicsStatus) // Attributes to change - attrs := &mlnxNic{totalVfs: -1} + attrs := &mlx.MlxNic{TotalVfs: -1} var changeWithoutReboot bool var totalVfs int - totalVfs, needReboot, changeWithoutReboot = handleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort) - sriovEnNeedReboot, sriovEnChangeWithoutReboot := handleEnableSriov(totalVfs, fwCurrent, fwNext, attrs) + totalVfs, needReboot, changeWithoutReboot = mlx.HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, isDualPort, mellanoxNicsSpec) + sriovEnNeedReboot, sriovEnChangeWithoutReboot := mlx.HandleEnableSriov(totalVfs, fwCurrent, fwNext, attrs) needReboot = needReboot || sriovEnNeedReboot changeWithoutReboot = changeWithoutReboot || sriovEnChangeWithoutReboot @@ -133,10 +116,10 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS if ifaceSpec.ExternallyManaged && needReboot { return true, true, fmt.Errorf( "interface %s required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf %d requested TotalVf %d", - ifaceSpec.PciAddress, fwCurrent.totalVfs, totalVfs) + ifaceSpec.PciAddress, fwCurrent.TotalVfs, totalVfs) } - needLinkChange, err := handleLinkType(pciPrefix, fwCurrent, attrs) + needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus) if err != nil { return false, false, err } @@ -162,14 +145,14 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS continue } - _, fwNext, err := getMlnxNicFwData(pciAddress) + _, fwNext, err := p.helpers.GetMlxNicFwData(pciAddress) if err != nil { return false, false, err } - if fwNext.totalVfs > 0 || fwNext.enableSriov { - attributesToChange[pciAddress] = mlnxNic{totalVfs: 0} - log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.totalVfs) + if fwNext.TotalVfs > 0 || fwNext.EnableSriov { + attributesToChange[pciAddress] = mlx.MlxNic{TotalVfs: 0} + log.Log.V(2).Info("Changing TotalVfs to 0, doesn't require rebooting", "fwNext.totalVfs", fwNext.TotalVfs) } } @@ -182,251 +165,10 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS // Apply config change func (p *MellanoxPlugin) Apply() error { - if utils.IsKernelLockdownMode(false) { + if p.helpers.IsKernelLockdownMode() { log.Log.Info("mellanox-plugin Apply() - skipping due to lockdown mode") return nil } log.Log.Info("mellanox-plugin Apply()") - return configFW() -} - -func configFW() error { - log.Log.Info("mellanox-plugin configFW()") - for pciAddr, fwArgs := range attributesToChange { - cmdArgs := []string{"-d", pciAddr, "-y", "set"} - if fwArgs.enableSriov { - cmdArgs = append(cmdArgs, fmt.Sprintf("%s=True", EnableSriov)) - } else if fwArgs.totalVfs == 0 { - cmdArgs = append(cmdArgs, fmt.Sprintf("%s=False", EnableSriov)) - } - if fwArgs.totalVfs > -1 { - cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%d", TotalVfs, fwArgs.totalVfs)) - } - if len(fwArgs.linkTypeP1) > 0 { - cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP1, fwArgs.linkTypeP1)) - } - if len(fwArgs.linkTypeP2) > 0 { - cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP2, fwArgs.linkTypeP2)) - } - - log.Log.V(2).Info("mellanox-plugin: configFW()", "cmd-args", cmdArgs) - if len(cmdArgs) <= 4 { - continue - } - _, err := utils.RunCommand("mstconfig", cmdArgs...) - if err != nil { - log.Log.Error(err, "mellanox-plugin configFW(): failed") - return err - } - } - return nil -} - -func getMlnxNicFwData(pciAddress string) (current, next *mlnxNic, err error) { - log.Log.Info("mellanox-plugin getMlnxNicFwData()", "device", pciAddress) - err = nil - attrs := []string{TotalVfs, EnableSriov, LinkTypeP1, LinkTypeP2} - - out, err := utils.MstConfigReadData(pciAddress) - if err != nil { - log.Log.Error(err, "mellanox-plugin getMlnxNicFwData(): failed") - return - } - mstCurrentData, mstNextData := utils.ParseMstconfigOutput(out, attrs) - current, err = mlnxNicFromMap(mstCurrentData) - if err != nil { - log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for current mstconfig data failed") - return - } - next, err = mlnxNicFromMap(mstNextData) - if err != nil { - log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for next mstconfig data failed") - } - return -} - -func mlnxNicFromMap(mstData map[string]string) (*mlnxNic, error) { - log.Log.Info("mellanox-plugin mlnxNicFromMap()", "data", mstData) - fwData := &mlnxNic{} - if strings.Contains(mstData[EnableSriov], "True") { - fwData.enableSriov = true - } - i, err := strconv.Atoi(mstData[TotalVfs]) - if err != nil { - return nil, err - } - - fwData.totalVfs = i - fwData.linkTypeP1 = getLinkType(mstData[LinkTypeP1]) - if linkTypeP2, ok := mstData[LinkTypeP2]; ok { - fwData.linkTypeP2 = getLinkType(linkTypeP2) - } - - return fwData, nil -} - -func getPciAddressPrefix(pciAddress string) string { - return pciAddress[:len(pciAddress)-1] -} - -func isDualPort(pciAddress string) bool { - log.Log.Info("mellanox-plugin isDualPort()", "address", pciAddress) - pciAddressPrefix := getPciAddressPrefix(pciAddress) - return len(mellanoxNicsStatus[pciAddressPrefix]) > 1 -} - -func getLinkType(linkType string) string { - log.Log.Info("mellanox-plugin getLinkType()", "link-type", linkType) - if strings.Contains(linkType, constants.LinkTypeETH) { - return constants.LinkTypeETH - } else if strings.Contains(linkType, constants.LinkTypeIB) { - return constants.LinkTypeIB - } else if len(linkType) > 0 { - log.Log.Error(nil, "mellanox-plugin getLinkType(): link type is not one of [ETH, IB], treating as unknown", - "link-type", linkType) - return UknownLinkType - } else { - log.Log.Info("mellanox-plugin getLinkType(): LINK_TYPE_P* attribute was not found, treating as preconfigured link type") - return PreconfiguredLinkType - } -} - -func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt, fwLinkType string) (bool, error) { - log.Log.Info("mellanox-plugin isLinkTypeRequireChange()", "device", iface.PciAddress) - if iface.LinkType != "" && !strings.EqualFold(ifaceStatus.LinkType, iface.LinkType) { - if !strings.EqualFold(iface.LinkType, constants.LinkTypeETH) && !strings.EqualFold(iface.LinkType, constants.LinkTypeIB) { - return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Not supported link type: %s,"+ - " supported link types: [eth, ETH, ib, and IB]", iface.LinkType) - } - if fwLinkType == UknownLinkType { - return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Unknown link type: %s", fwLinkType) - } - if fwLinkType == PreconfiguredLinkType { - return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Network card %s does not support link type change", iface.PciAddress) - } - - return true, nil - } - - return false, nil -} - -func getOtherPortSpec(pciAddress string) *sriovnetworkv1.Interface { - log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress) - pciAddrPrefix := getPciAddressPrefix(pciAddress) - pciAddrSuffix := pciAddress[len(pciAddrPrefix):] - - if pciAddrSuffix == "0" { - iface := mellanoxNicsSpec[pciAddrPrefix+"1"] - return &iface - } - - iface := mellanoxNicsSpec[pciAddrPrefix+"0"] - return &iface -} - -// handleTotalVfs return required total VFs or max (required VFs for dual port NIC) and needReboot if totalVfs will change -func handleTotalVfs(fwCurrent, fwNext, attrs *mlnxNic, ifaceSpec sriovnetworkv1.Interface, isDualPort bool) ( - totalVfs int, needReboot, changeWithoutReboot bool) { - totalVfs = ifaceSpec.NumVfs - // Check if the other port is changing the number of VF - if isDualPort { - otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress) - if otherIfaceSpec != nil { - if otherIfaceSpec.NumVfs > totalVfs { - totalVfs = otherIfaceSpec.NumVfs - } - } - } - - // if the PF is externally managed we just need to check the totalVfs requested in the policy is not higher than - // the configured amount - if ifaceSpec.ExternallyManaged { - if totalVfs > fwCurrent.totalVfs { - log.Log.Error(nil, "The nic is externallyManaged and TotalVfs configured on the system is lower then requested VFs, failing configuration", - "current", fwCurrent.totalVfs, "requested", totalVfs) - attrs.totalVfs = totalVfs - needReboot = true - changeWithoutReboot = false - } - return - } - - if fwCurrent.totalVfs != totalVfs { - log.Log.V(2).Info("Changing TotalVfs, needs reboot", "current", fwCurrent.totalVfs, "requested", totalVfs) - attrs.totalVfs = totalVfs - needReboot = true - } - - // Remove policy then re-apply it - if !needReboot && fwNext.totalVfs != totalVfs { - log.Log.V(2).Info("Changing TotalVfs to same as Next Boot value, doesn't require rebooting", - "current", fwCurrent.totalVfs, "next", fwNext.totalVfs, "requested", totalVfs) - attrs.totalVfs = totalVfs - changeWithoutReboot = true - } - - return -} - -// handleEnableSriov based on totalVfs it decide to disable (totalVfs=0) or enable (totalVfs changed from 0) sriov -// and need reboot if enableSriov will change -func handleEnableSriov(totalVfs int, fwCurrent, fwNext, attrs *mlnxNic) (needReboot, changeWithoutReboot bool) { - if totalVfs == 0 && fwCurrent.enableSriov { - log.Log.V(2).Info("disabling Sriov, needs reboot") - attrs.enableSriov = false - return true, false - } else if totalVfs > 0 && !fwCurrent.enableSriov { - log.Log.V(2).Info("enabling Sriov, needs reboot") - attrs.enableSriov = true - return true, false - } else if totalVfs > 0 && !fwNext.enableSriov { - attrs.enableSriov = true - return false, true - } - - return false, false -} - -func getIfaceStatus(pciAddress string) sriovnetworkv1.InterfaceExt { - return mellanoxNicsStatus[getPciAddressPrefix(pciAddress)][pciAddress] -} - -// handleLinkType based on existing linkType and requested link -func handleLinkType(pciPrefix string, fwData, attr *mlnxNic) (bool, error) { - needReboot := false - - pciAddress := pciPrefix + "0" - if firstPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok { - ifaceStatus := getIfaceStatus(pciAddress) - needChange, err := isLinkTypeRequireChange(firstPortSpec, ifaceStatus, fwData.linkTypeP1) - if err != nil { - return false, err - } - - if needChange { - log.Log.V(2).Info("Changing LinkTypeP1, needs reboot", - "from", fwData.linkTypeP1, "to", firstPortSpec.LinkType) - attr.linkTypeP1 = firstPortSpec.LinkType - needReboot = true - } - } - - pciAddress = pciPrefix + "1" - if secondPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok { - ifaceStatus := getIfaceStatus(pciAddress) - needChange, err := isLinkTypeRequireChange(secondPortSpec, ifaceStatus, fwData.linkTypeP2) - if err != nil { - return false, err - } - - if needChange { - log.Log.V(2).Info("Changing LinkTypeP2, needs reboot", - "from", fwData.linkTypeP2, "to", secondPortSpec.LinkType) - attr.linkTypeP2 = secondPortSpec.LinkType - needReboot = true - } - } - - return needReboot, nil + return p.helpers.MlxConfigFW(attributesToChange) } diff --git a/pkg/plugins/mock/mock_plugin.go b/pkg/plugins/mock/mock_plugin.go new file mode 100644 index 000000000..b821139c5 --- /dev/null +++ b/pkg/plugins/mock/mock_plugin.go @@ -0,0 +1,93 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: plugin.go + +// Package mock_plugin is a generated GoMock package. +package mock_plugin + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +// MockVendorPlugin is a mock of VendorPlugin interface. +type MockVendorPlugin struct { + ctrl *gomock.Controller + recorder *MockVendorPluginMockRecorder +} + +// MockVendorPluginMockRecorder is the mock recorder for MockVendorPlugin. +type MockVendorPluginMockRecorder struct { + mock *MockVendorPlugin +} + +// NewMockVendorPlugin creates a new mock instance. +func NewMockVendorPlugin(ctrl *gomock.Controller) *MockVendorPlugin { + mock := &MockVendorPlugin{ctrl: ctrl} + mock.recorder = &MockVendorPluginMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVendorPlugin) EXPECT() *MockVendorPluginMockRecorder { + return m.recorder +} + +// Apply mocks base method. +func (m *MockVendorPlugin) Apply() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Apply") + ret0, _ := ret[0].(error) + return ret0 +} + +// Apply indicates an expected call of Apply. +func (mr *MockVendorPluginMockRecorder) Apply() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockVendorPlugin)(nil).Apply)) +} + +// Name mocks base method. +func (m *MockVendorPlugin) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockVendorPluginMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockVendorPlugin)(nil).Name)) +} + +// OnNodeStateChange mocks base method. +func (m *MockVendorPlugin) OnNodeStateChange(new *v1.SriovNetworkNodeState) (bool, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OnNodeStateChange", new) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// OnNodeStateChange indicates an expected call of OnNodeStateChange. +func (mr *MockVendorPluginMockRecorder) OnNodeStateChange(new interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnNodeStateChange", reflect.TypeOf((*MockVendorPlugin)(nil).OnNodeStateChange), new) +} + +// Spec mocks base method. +func (m *MockVendorPlugin) Spec() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Spec") + ret0, _ := ret[0].(string) + return ret0 +} + +// Spec indicates an expected call of Spec. +func (mr *MockVendorPluginMockRecorder) Spec() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Spec", reflect.TypeOf((*MockVendorPlugin)(nil).Spec)) +} diff --git a/pkg/plugins/plugin.go b/pkg/plugins/plugin.go index 1723ca610..276e37cf2 100644 --- a/pkg/plugins/plugin.go +++ b/pkg/plugins/plugin.go @@ -4,6 +4,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" ) +//go:generate ../../bin/mockgen -destination mock/mock_plugin.go -source plugin.go type VendorPlugin interface { // Return the name of plugin Name() string diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go index 9fb6cb774..06a57a80f 100644 --- a/pkg/plugins/virtual/virtual_plugin.go +++ b/pkg/plugins/virtual/virtual_plugin.go @@ -6,10 +6,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + consts "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) var PluginName = "virtual_plugin" @@ -21,8 +21,7 @@ type VirtualPlugin struct { DesireState *sriovnetworkv1.SriovNetworkNodeState LastState *sriovnetworkv1.SriovNetworkNodeState LoadVfioDriver uint - RunningOnHost bool - HostManager host.HostManagerInterface + helpers helper.HostHelpersInterface } const ( @@ -32,13 +31,12 @@ const ( ) // Initialize our plugin and set up initial values -func NewVirtualPlugin(runningOnHost bool) (plugin.VendorPlugin, error) { +func NewVirtualPlugin(helper helper.HostHelpersInterface) (plugin.VendorPlugin, error) { return &VirtualPlugin{ PluginName: PluginName, SpecVersion: "1.0", LoadVfioDriver: unloaded, - RunningOnHost: runningOnHost, - HostManager: host.NewHostManager(runningOnHost), + helpers: helper, }, nil } @@ -79,12 +77,12 @@ func (p *VirtualPlugin) Apply() error { // This is the case for OpenStack deployments where the underlying virtualization platform is KVM. // NOTE: if VFIO was already loaded for some reason, we will not try to load it again with the new options. kernelArgs := "enable_unsafe_noiommu_mode=1" - if err := p.HostManager.LoadKernelModule("vfio", kernelArgs); err != nil { + if err := p.helpers.LoadKernelModule("vfio", kernelArgs); err != nil { log.Log.Error(err, "virtual-plugin Apply(): fail to load vfio kmod") return err } - if err := p.HostManager.LoadKernelModule("vfio_pci"); err != nil { + if err := p.helpers.LoadKernelModule("vfio_pci"); err != nil { log.Log.Error(err, "virtual-plugin Apply(): fail to load vfio_pci kmod") return err } @@ -98,12 +96,12 @@ func (p *VirtualPlugin) Apply() error { return nil } } - exit, err := utils.Chroot("/host") + exit, err := p.helpers.Chroot(consts.Host) if err != nil { return err } defer exit() - if err := utils.SyncNodeStateVirtual(p.DesireState); err != nil { + if err := syncNodeStateVirtual(p.DesireState, p.helpers); err != nil { return err } p.LastState = &sriovnetworkv1.SriovNetworkNodeState{} @@ -122,7 +120,62 @@ func (p *VirtualPlugin) IsSystemService() bool { func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool { for _, iface := range state.Spec.Interfaces { for i := range iface.VfGroups { - if iface.VfGroups[i].DeviceType == constants.DeviceTypeVfioPci { + if iface.VfGroups[i].DeviceType == consts.DeviceTypeVfioPci { + return true + } + } + } + return false +} + +// syncNodeStateVirtual attempt to update the node state to match the desired state in virtual platforms +func syncNodeStateVirtual(newState *sriovnetworkv1.SriovNetworkNodeState, helpers helper.HostHelpersInterface) error { + var err error + for _, ifaceStatus := range newState.Status.Interfaces { + for _, iface := range newState.Spec.Interfaces { + if iface.PciAddress == ifaceStatus.PciAddress { + if !needUpdateVirtual(&iface, &ifaceStatus) { + log.Log.V(2).Info("syncNodeStateVirtual(): no need update interface", "address", iface.PciAddress) + break + } + if err = helpers.ConfigSriovDeviceVirtual(&iface); err != nil { + log.Log.Error(err, "syncNodeStateVirtual(): fail to config sriov interface", "address", iface.PciAddress) + return err + } + break + } + } + } + return nil +} + +func needUpdateVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) bool { + // The device MTU is set by the platform + // The NumVfs is always 1 + if iface.NumVfs > 0 { + for _, vf := range ifaceStatus.VFs { + ingroup := false + for _, group := range iface.VfGroups { + if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) { + ingroup = true + if group.DeviceType != consts.DeviceTypeNetDevice { + if group.DeviceType != vf.Driver { + log.Log.V(2).Info("needUpdateVirtual(): Driver needs update", + "desired", group.DeviceType, "current", vf.Driver) + return true + } + } else { + if sriovnetworkv1.StringInArray(vf.Driver, vars.DpdkDrivers) { + log.Log.V(2).Info("needUpdateVirtual(): Driver needs update", + "desired", group.DeviceType, "current", vf.Driver) + return true + } + } + break + } + } + if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, vars.DpdkDrivers) { + // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver. return true } } diff --git a/pkg/service/service.go b/pkg/service/service.go deleted file mode 100644 index 671d2e20a..000000000 --- a/pkg/service/service.go +++ /dev/null @@ -1,15 +0,0 @@ -package service - -type Service struct { - Name string - Path string - Content string -} - -func NewService(name, path, content string) *Service { - return &Service{ - Name: name, - Path: path, - Content: content, - } -} diff --git a/pkg/service/service_manager.go b/pkg/service/service_manager.go deleted file mode 100644 index 29653fe10..000000000 --- a/pkg/service/service_manager.go +++ /dev/null @@ -1,94 +0,0 @@ -package service - -import ( - "os" - "os/exec" - "path" - "path/filepath" - - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -type ServiceManager interface { - IsServiceExist(string) (bool, error) - IsServiceEnabled(string) (bool, error) - ReadService(string) (*Service, error) - EnableService(service *Service) error -} - -type serviceManager struct { - chroot string -} - -func NewServiceManager(chroot string) ServiceManager { - root := chroot - if root == "" { - root = "/" - } - return &serviceManager{root} -} - -// IsServiceExist check if service unit exist -func (sm *serviceManager) IsServiceExist(servicePath string) (bool, error) { - _, err := os.Stat(path.Join(sm.chroot, servicePath)) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - - return true, nil -} - -// IsServiceEnabled check if service exist and enabled -func (sm *serviceManager) IsServiceEnabled(servicePath string) (bool, error) { - exist, err := sm.IsServiceExist(servicePath) - if err != nil || !exist { - return false, err - } - serviceName := filepath.Base(servicePath) - // Change root dir - exit, err := utils.Chroot(sm.chroot) - if err != nil { - return false, err - } - defer exit() - - cmd := exec.Command("systemctl", "is-enabled", serviceName) - return cmd.Run() == nil, nil -} - -// ReadService read service from given path -func (sm *serviceManager) ReadService(servicePath string) (*Service, error) { - data, err := os.ReadFile(path.Join(sm.chroot, servicePath)) - if err != nil { - return nil, err - } - - return &Service{ - Name: filepath.Base(servicePath), - Path: servicePath, - Content: string(data), - }, nil -} - -// EnableService creates service file and enables it with systemctl enable -func (sm *serviceManager) EnableService(service *Service) error { - // Write service file - err := os.WriteFile(path.Join(sm.chroot, service.Path), []byte(service.Content), 0644) - if err != nil { - return err - } - - // Change root dir - exit, err := utils.Chroot(sm.chroot) - if err != nil { - return err - } - defer exit() - - // Enable service - cmd := exec.Command("systemctl", "enable", service.Name) - return cmd.Run() -} diff --git a/pkg/service/types.go b/pkg/service/types.go deleted file mode 100644 index aaa879a4d..000000000 --- a/pkg/service/types.go +++ /dev/null @@ -1,23 +0,0 @@ -package service - -// ServiceInjectionManifestFile service injection manifest file structure -type ServiceInjectionManifestFile struct { - Name string - Dropins []struct { - Contents string - } -} - -// ServiceManifestFile service manifest file structure -type ServiceManifestFile struct { - Name string - Contents string -} - -// ScriptManifestFile script manifest file structure -type ScriptManifestFile struct { - Path string - Contents struct { - Inline string - } -} diff --git a/pkg/service/utils.go b/pkg/service/utils.go deleted file mode 100644 index 4abeae553..000000000 --- a/pkg/service/utils.go +++ /dev/null @@ -1,151 +0,0 @@ -package service - -import ( - "io" - "os" - "strings" - - "github.com/coreos/go-systemd/v22/unit" - "gopkg.in/yaml.v2" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -const systemdDir = "/usr/lib/systemd/system/" - -// CompareServices compare 2 service and return true if serviceA has all the fields of serviceB -func CompareServices(serviceA, serviceB *Service) (bool, error) { - optsA, err := unit.Deserialize(strings.NewReader(serviceA.Content)) - if err != nil { - return false, err - } - optsB, err := unit.Deserialize(strings.NewReader(serviceB.Content)) - if err != nil { - return false, err - } - -OUTER: - for _, optB := range optsB { - for _, optA := range optsA { - if optA.Match(optB) { - continue OUTER - } - } - log.Log.V(2).Info("CompareServices", "ServiceA", optsA, "ServiceB", *optB) - return true, nil - } - - return false, nil -} - -// RemoveFromService removes given fields from service -func RemoveFromService(service *Service, options ...*unit.UnitOption) (*Service, error) { - opts, err := unit.Deserialize(strings.NewReader(service.Content)) - if err != nil { - return nil, err - } - - var newServiceOptions []*unit.UnitOption -OUTER: - for _, opt := range opts { - for _, optRemove := range options { - if opt.Match(optRemove) { - continue OUTER - } - } - - newServiceOptions = append(newServiceOptions, opt) - } - - data, err := io.ReadAll(unit.Serialize(newServiceOptions)) - if err != nil { - return nil, err - } - - return &Service{ - Name: service.Name, - Path: service.Path, - Content: string(data), - }, nil -} - -// AppendToService appends given fields to service -func AppendToService(service *Service, options ...*unit.UnitOption) (*Service, error) { - serviceOptions, err := unit.Deserialize(strings.NewReader(service.Content)) - if err != nil { - return nil, err - } - -OUTER: - for _, appendOpt := range options { - for _, opt := range serviceOptions { - if opt.Match(appendOpt) { - continue OUTER - } - } - serviceOptions = append(serviceOptions, appendOpt) - } - - data, err := io.ReadAll(unit.Serialize(serviceOptions)) - if err != nil { - return nil, err - } - - return &Service{ - Name: service.Name, - Path: service.Path, - Content: string(data), - }, nil -} - -// ReadServiceInjectionManifestFile reads service injection file -func ReadServiceInjectionManifestFile(path string) (*Service, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var serviceContent ServiceInjectionManifestFile - if err := yaml.Unmarshal(data, &serviceContent); err != nil { - return nil, err - } - - return &Service{ - Name: serviceContent.Name, - Path: systemdDir + serviceContent.Name, - Content: serviceContent.Dropins[0].Contents, - }, nil -} - -// ReadServiceManifestFile reads service file -func ReadServiceManifestFile(path string) (*Service, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var serviceFile *ServiceManifestFile - if err := yaml.Unmarshal(data, &serviceFile); err != nil { - return nil, err - } - - return &Service{ - Name: serviceFile.Name, - Path: "/etc/systemd/system/" + serviceFile.Name, - Content: serviceFile.Contents, - }, nil -} - -// ReadScriptManifestFile reads script file -func ReadScriptManifestFile(path string) (*ScriptManifestFile, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var scriptFile *ScriptManifestFile - if err := yaml.Unmarshal(data, &scriptFile); err != nil { - return nil, err - } - - return scriptFile, nil -} diff --git a/pkg/systemd/systemd.go b/pkg/systemd/systemd.go index 87ae19a11..f682d85f5 100644 --- a/pkg/systemd/systemd.go +++ b/pkg/systemd/systemd.go @@ -25,30 +25,33 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) const ( - SriovSystemdConfigPath = utils.SriovConfBasePath + "/sriov-interface-config.yaml" - SriovSystemdResultPath = utils.SriovConfBasePath + "/sriov-interface-result.yaml" - sriovSystemdSupportedNicPath = utils.SriovConfBasePath + "/sriov-supported-nics-ids.yaml" + SriovSystemdConfigPath = consts.SriovConfBasePath + "/sriov-interface-config.yaml" + SriovSystemdResultPath = consts.SriovConfBasePath + "/sriov-interface-result.yaml" + sriovSystemdSupportedNicPath = consts.SriovConfBasePath + "/sriov-supported-nics-ids.yaml" sriovSystemdServiceBinaryPath = "/var/lib/sriov/sriov-network-config-daemon" - SriovHostSystemdConfigPath = "/host" + SriovSystemdConfigPath - SriovHostSystemdResultPath = "/host" + SriovSystemdResultPath - sriovHostSystemdSupportedNicPath = "/host" + sriovSystemdSupportedNicPath - sriovHostSystemdServiceBinaryPath = "/host" + sriovSystemdServiceBinaryPath + SriovHostSystemdConfigPath = consts.Host + SriovSystemdConfigPath + SriovHostSystemdResultPath = consts.Host + SriovSystemdResultPath + sriovHostSystemdSupportedNicPath = consts.Host + sriovSystemdSupportedNicPath + sriovHostSystemdServiceBinaryPath = consts.Host + sriovSystemdServiceBinaryPath SriovServicePath = "/etc/systemd/system/sriov-config.service" - SriovHostServicePath = "/host" + SriovServicePath + SriovHostServicePath = consts.Host + SriovServicePath - HostSriovConfBasePath = "/host" + utils.SriovConfBasePath + HostSriovConfBasePath = consts.Host + consts.SriovConfBasePath ) +// TODO: move this to the host interface also + type SriovConfig struct { Spec sriovnetworkv1.SriovNetworkNodeStateSpec `yaml:"spec"` UnsupportedNics bool `yaml:"unsupportedNics"` - PlatformType utils.PlatformType `yaml:"platformType"` + PlatformType consts.PlatformTypes `yaml:"platformType"` } type SriovResult struct { @@ -67,15 +70,15 @@ func ReadConfFile() (spec *SriovConfig, err error) { return spec, err } -func WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState, unsupportedNics bool, platformType utils.PlatformType) (bool, error) { +func WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { newFile := false // remove the device plugin revision as we don't need it here newState.Spec.DpConfigVersion = "" sriovConfig := &SriovConfig{ newState.Spec, - unsupportedNics, - platformType, + vars.DevMode, + vars.PlatformType, } _, err := os.Stat(SriovHostSystemdConfigPath) diff --git a/pkg/utils/cluster.go b/pkg/utils/cluster.go index 610bab415..13310347f 100644 --- a/pkg/utils/cluster.go +++ b/pkg/utils/cluster.go @@ -11,6 +11,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" ) const ( @@ -35,7 +37,7 @@ func getNodeRole(node corev1.Node) string { } func IsSingleNodeCluster(c client.Client) (bool, error) { - if os.Getenv("CLUSTER_TYPE") == ClusterTypeOpenshift { + if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeOpenshift { topo, err := openshiftControlPlaneTopologyStatus(c) if err != nil { return false, err @@ -53,7 +55,7 @@ func IsSingleNodeCluster(c client.Client) (bool, error) { // On kubernetes, it is determined by which node the sriov operator is scheduled on. If operator // pod is schedule on worker node, it is considered as external control plane. func IsExternalControlPlaneCluster(c client.Client) (bool, error) { - if os.Getenv("CLUSTER_TYPE") == ClusterTypeOpenshift { + if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeOpenshift { topo, err := openshiftControlPlaneTopologyStatus(c) if err != nil { return false, err @@ -61,7 +63,7 @@ func IsExternalControlPlaneCluster(c client.Client) (bool, error) { if topo == "External" { return true, nil } - } else if os.Getenv("CLUSTER_TYPE") == ClusterTypeKubernetes { + } else if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeKubernetes { role, err := operatorNodeRole(c) if err != nil { return false, err diff --git a/pkg/utils/command.go b/pkg/utils/command.go deleted file mode 100644 index 085cf5881..000000000 --- a/pkg/utils/command.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package utils - -import ( - "bytes" - "os/exec" -) - -// Interface to run commands -// -//go:generate ../../bin/mockgen -destination mock/mock_command.go -source command.go -type CommandInterface interface { - Run(string, ...string) (stdout bytes.Buffer, stderr bytes.Buffer, err error) -} - -type Command struct { -} - -func (c *Command) Run(name string, args ...string) (stdout bytes.Buffer, stderr bytes.Buffer, err error) { - var stdoutbuff, stderrbuff bytes.Buffer - cmd := exec.Command(name, args...) - cmd.Stdout = &stdoutbuff - cmd.Stderr = &stderrbuff - - err = cmd.Run() - return -} diff --git a/pkg/utils/driver.go b/pkg/utils/driver.go deleted file mode 100644 index c91c6448e..000000000 --- a/pkg/utils/driver.go +++ /dev/null @@ -1,119 +0,0 @@ -package utils - -import ( - "fmt" - "os" - "path/filepath" - - dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" - "sigs.k8s.io/controller-runtime/pkg/log" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -var DpdkDrivers = []string{"igb_uio", "vfio-pci", "uio_pci_generic"} - -// Unbind unbind driver for one device -func Unbind(pciAddr string) error { - log.Log.V(2).Info("Unbind(): unbind device driver for device", "device", pciAddr) - yes, driver := hasDriver(pciAddr) - if !yes { - return nil - } - - filePath := filepath.Join(sysBusPciDrivers, driver, "unbind") - err := os.WriteFile(filePath, []byte(pciAddr), os.ModeAppend) - if err != nil { - log.Log.Error(err, "Unbind(): fail to unbind driver for device", "device", pciAddr) - return err - } - return nil -} - -// BindDpdkDriver bind dpdk driver for one device -// Bind the device given by "pciAddr" to the driver "driver" -func BindDpdkDriver(pciAddr, driver string) error { - log.Log.V(2).Info("BindDpdkDriver(): bind device to driver", - "device", pciAddr, "driver", driver) - - if yes, d := hasDriver(pciAddr); yes { - if driver == d { - log.Log.V(2).Info("BindDpdkDriver(): device already bound to driver", - "device", pciAddr, "driver", driver) - return nil - } - - if err := Unbind(pciAddr); err != nil { - return err - } - } - - driverOverridePath := filepath.Join(sysBusPciDevices, pciAddr, "driver_override") - err := os.WriteFile(driverOverridePath, []byte(driver), os.ModeAppend) - if err != nil { - log.Log.Error(err, "BindDpdkDriver(): fail to write driver_override for device", - "device", pciAddr, "driver", driver) - return err - } - bindPath := filepath.Join(sysBusPciDrivers, driver, "bind") - err = os.WriteFile(bindPath, []byte(pciAddr), os.ModeAppend) - if err != nil { - log.Log.Error(err, "BindDpdkDriver(): fail to bind driver for device", - "driver", driver, "device", pciAddr) - _, err := os.Readlink(filepath.Join(sysBusPciDevices, pciAddr, "iommu_group")) - if err != nil { - log.Log.Error(err, "Could not read IOMMU group for device", "device", pciAddr) - return fmt.Errorf( - "cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, err) - } - return err - } - err = os.WriteFile(driverOverridePath, []byte(""), os.ModeAppend) - if err != nil { - log.Log.Error(err, "BindDpdkDriver(): failed to clear driver_override for device", "device", pciAddr) - return err - } - - return nil -} - -// BindDefaultDriver bind driver for one device -// Bind the device given by "pciAddr" to the default driver -func BindDefaultDriver(pciAddr string) error { - log.Log.V(2).Info("BindDefaultDriver(): bind device to default driver", "device", pciAddr) - - if yes, d := hasDriver(pciAddr); yes { - if !sriovnetworkv1.StringInArray(d, DpdkDrivers) { - log.Log.V(2).Info("BindDefaultDriver(): device already bound to default driver", - "device", pciAddr, "driver", d) - return nil - } - if err := Unbind(pciAddr); err != nil { - return err - } - } - - driverOverridePath := filepath.Join(sysBusPciDevices, pciAddr, "driver_override") - err := os.WriteFile(driverOverridePath, []byte("\x00"), os.ModeAppend) - if err != nil { - log.Log.Error(err, "BindDefaultDriver(): failed to write driver_override for device", "device", pciAddr) - return err - } - err = os.WriteFile(sysBusPciDriversProbe, []byte(pciAddr), os.ModeAppend) - if err != nil { - log.Log.Error(err, "BindDefaultDriver(): failed to bind driver for device", "device", pciAddr) - return err - } - - return nil -} - -func hasDriver(pciAddr string) (bool, string) { - driver, err := dputils.GetDriverName(pciAddr) - if err != nil { - log.Log.V(2).Info("hasDriver(): device driver is empty for device", "device", pciAddr) - return false, "" - } - log.Log.V(2).Info("hasDriver(): device driver for device", "device", pciAddr, "driver", driver) - return true, driver -} diff --git a/pkg/utils/mock/mock_utils.go b/pkg/utils/mock/mock_utils.go new file mode 100644 index 000000000..636c7f4a1 --- /dev/null +++ b/pkg/utils/mock/mock_utils.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: utils.go + +// Package mock_utils is a generated GoMock package. +package mock_utils + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockCmdInterface is a mock of CmdInterface interface. +type MockCmdInterface struct { + ctrl *gomock.Controller + recorder *MockCmdInterfaceMockRecorder +} + +// MockCmdInterfaceMockRecorder is the mock recorder for MockCmdInterface. +type MockCmdInterfaceMockRecorder struct { + mock *MockCmdInterface +} + +// NewMockCmdInterface creates a new mock instance. +func NewMockCmdInterface(ctrl *gomock.Controller) *MockCmdInterface { + mock := &MockCmdInterface{ctrl: ctrl} + mock.recorder = &MockCmdInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCmdInterface) EXPECT() *MockCmdInterfaceMockRecorder { + return m.recorder +} + +// Chroot mocks base method. +func (m *MockCmdInterface) Chroot(arg0 string) (func() error, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Chroot", arg0) + ret0, _ := ret[0].(func() error) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Chroot indicates an expected call of Chroot. +func (mr *MockCmdInterfaceMockRecorder) Chroot(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Chroot", reflect.TypeOf((*MockCmdInterface)(nil).Chroot), arg0) +} + +// RunCommand mocks base method. +func (m *MockCmdInterface) RunCommand(arg0 string, arg1 ...string) (string, string, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RunCommand", varargs...) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// RunCommand indicates an expected call of RunCommand. +func (mr *MockCmdInterfaceMockRecorder) RunCommand(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunCommand", reflect.TypeOf((*MockCmdInterface)(nil).RunCommand), varargs...) +} diff --git a/pkg/utils/shutdown.go b/pkg/utils/shutdown.go index 3700efa18..208e2073c 100644 --- a/pkg/utils/shutdown.go +++ b/pkg/utils/shutdown.go @@ -11,7 +11,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" ) var shutdownLog = ctrl.Log.WithName("shutdown") @@ -68,7 +68,7 @@ func updateWebhooks() { func updateValidatingWebhook(c *kubernetes.Clientset) { validatingWebhookClient := c.AdmissionregistrationV1().ValidatingWebhookConfigurations() - webhook, err := validatingWebhookClient.Get(context.TODO(), constants.OperatorWebHookName, metav1.GetOptions{}) + webhook, err := validatingWebhookClient.Get(context.TODO(), consts.OperatorWebHookName, metav1.GetOptions{}) if err != nil { shutdownLog.Error(err, "Error getting webhook") } @@ -81,7 +81,7 @@ func updateValidatingWebhook(c *kubernetes.Clientset) { func updateMutatingWebhooks(c *kubernetes.Clientset) { mutatingWebhookClient := c.AdmissionregistrationV1().MutatingWebhookConfigurations() - for _, name := range []string{constants.OperatorWebHookName, constants.InjectorWebHookName} { + for _, name := range []string{consts.OperatorWebHookName, consts.InjectorWebHookName} { mutatingWebhook, err := mutatingWebhookClient.Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { shutdownLog.Error(err, "Error getting webhook") diff --git a/pkg/utils/sriov.go b/pkg/utils/sriov.go deleted file mode 100644 index f0668494f..000000000 --- a/pkg/utils/sriov.go +++ /dev/null @@ -1,151 +0,0 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package utils - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - - "sigs.k8s.io/controller-runtime/pkg/log" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -const ( - SriovSwitchDevConfPath = SriovConfBasePath + "/sriov_config.json" - SriovHostSwitchDevConfPath = "/host" + SriovSwitchDevConfPath -) - -type config struct { - Interfaces []sriovnetworkv1.Interface `json:"interfaces"` -} - -func IsSwitchdevModeSpec(spec sriovnetworkv1.SriovNetworkNodeStateSpec) bool { - for _, iface := range spec.Interfaces { - if iface.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - return true - } - } - return false -} - -func findInterface(interfaces sriovnetworkv1.Interfaces, name string) (iface sriovnetworkv1.Interface, err error) { - for _, i := range interfaces { - if i.Name == name { - return i, nil - } - } - return sriovnetworkv1.Interface{}, fmt.Errorf("unable to find interface: %v", name) -} - -func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update bool, err error) { - // Create a map with all the PFs we will need to SKIP for systemd configuration - pfsToSkip, err := GetPfsToSkip(newState) - if err != nil { - return false, err - } - cfg := config{} - for _, iface := range newState.Spec.Interfaces { - for _, ifaceStatus := range newState.Status.Interfaces { - if iface.PciAddress != ifaceStatus.PciAddress { - continue - } - - if skip := pfsToSkip[iface.PciAddress]; !skip { - continue - } - - i := sriovnetworkv1.Interface{} - if iface.NumVfs > 0 { - var vfGroups []sriovnetworkv1.VfGroup = nil - ifc, err := findInterface(newState.Spec.Interfaces, iface.Name) - if err != nil { - log.Log.Error(err, "WriteSwitchdevConfFile(): fail find interface") - } else { - vfGroups = ifc.VfGroups - } - i = sriovnetworkv1.Interface{ - // Not passing all the contents, since only NumVfs and EswitchMode can be configured by configure-switchdev.sh currently. - Name: iface.Name, - PciAddress: iface.PciAddress, - NumVfs: iface.NumVfs, - Mtu: iface.Mtu, - VfGroups: vfGroups, - } - - if iface.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - i.EswitchMode = iface.EswitchMode - } - cfg.Interfaces = append(cfg.Interfaces, i) - } - } - } - _, err = os.Stat(SriovHostSwitchDevConfPath) - if err != nil { - if os.IsNotExist(err) { - if len(cfg.Interfaces) == 0 { - err = nil - return - } - - // Create the sriov-operator folder on the host if it doesn't exist - if _, err := os.Stat("/host" + SriovConfBasePath); os.IsNotExist(err) { - err = os.Mkdir("/host"+SriovConfBasePath, os.ModeDir) - if err != nil { - log.Log.Error(err, "WriteConfFile(): failed to create sriov-operator folder") - return false, err - } - } - - log.Log.V(2).Info("WriteSwitchdevConfFile(): file not existed, create it") - _, err = os.Create(SriovHostSwitchDevConfPath) - if err != nil { - log.Log.Error(err, "WriteSwitchdevConfFile(): failed to create file") - return - } - } else { - return - } - } - oldContent, err := os.ReadFile(SriovHostSwitchDevConfPath) - if err != nil { - log.Log.Error(err, "WriteSwitchdevConfFile(): failed to read file") - return - } - var newContent []byte - if len(cfg.Interfaces) != 0 { - newContent, err = json.Marshal(cfg) - if err != nil { - log.Log.Error(err, "WriteSwitchdevConfFile(): fail to marshal config") - return - } - } - - if bytes.Equal(newContent, oldContent) { - log.Log.V(2).Info("WriteSwitchdevConfFile(): no update") - return - } - update = true - log.Log.V(2).Info("WriteSwitchdevConfFile(): write to switchdev.conf", "content", newContent) - err = os.WriteFile(SriovHostSwitchDevConfPath, newContent, 0644) - if err != nil { - log.Log.Error(err, "WriteSwitchdevConfFile(): failed to write file") - return - } - return -} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 9452bce41..3a0bbdc4c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,741 +2,38 @@ package utils import ( "bytes" - "errors" + "encoding/hex" "fmt" + "hash/fnv" "math/rand" "net" "os" "os/exec" - "path" "path/filepath" - "regexp" - "strconv" - "strings" - "syscall" - "time" - - "encoding/hex" - "hash/fnv" "sort" + "syscall" corev1 "k8s.io/api/core/v1" - - "github.com/cenkalti/backoff" - "github.com/jaypipes/ghw" - "github.com/vishvananda/netlink" - "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/log" - dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -const ( - sysBusPciDevices = "/sys/bus/pci/devices" - sysBusPciDrivers = "/sys/bus/pci/drivers" - sysBusPciDriversProbe = "/sys/bus/pci/drivers_probe" - sysClassNet = "/sys/class/net" - procKernelCmdLine = "/proc/cmdline" - netClass = 0x02 - numVfsFile = "sriov_numvfs" - - ClusterTypeOpenshift = "openshift" - ClusterTypeKubernetes = "kubernetes" - VendorMellanox = "15b3" - DeviceBF2 = "a2d6" - DeviceBF3 = "a2dc" - - udevFolder = "/etc/udev" - udevRulesFolder = udevFolder + "/rules.d" - udevDisableNM = "/bindata/scripts/udev-find-sriov-pf.sh" - nmUdevRule = "SUBSYSTEM==\"net\", ACTION==\"add|change|move\", ATTRS{device}==\"%s\", IMPORT{program}=\"/etc/udev/disable-nm-sriov.sh $env{INTERFACE} %s\"" - - KernelArgPciRealloc = "pci=realloc" - KernelArgIntelIommu = "intel_iommu=on" - KernelArgIommuPt = "iommu=pt" -) - -var InitialState sriovnetworkv1.SriovNetworkNodeState -var ClusterType string - -var pfPhysPortNameRe = regexp.MustCompile(`p\d+`) - -// FilesystemRoot used by test to mock interactions with filesystem -var FilesystemRoot = "" - -var SupportedVfIds []string - -func init() { - ClusterType = os.Getenv("CLUSTER_TYPE") -} - -// GetCurrentKernelArgs This retrieves the kernel cmd line arguments -func GetCurrentKernelArgs(chroot bool) (string, error) { - path := procKernelCmdLine - if !chroot { - path = "/host" + path - } - cmdLine, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("GetCurrentKernelArgs(): Error reading %s: %v", procKernelCmdLine, err) - } - return string(cmdLine), nil -} - -// IsKernelArgsSet This checks if the kernel cmd line is set properly. Please note that the same key could be repeated -// several times in the kernel cmd line. We can only ensure that the kernel cmd line has the key/val kernel arg that we set. -func IsKernelArgsSet(cmdLine string, karg string) bool { - elements := strings.Fields(cmdLine) - for _, element := range elements { - if element == karg { - return true - } - } - return false -} - -func DiscoverSriovDevices(withUnsupported bool, storeManager StoreManagerInterface) ([]sriovnetworkv1.InterfaceExt, error) { - log.Log.V(2).Info("DiscoverSriovDevices") - pfList := []sriovnetworkv1.InterfaceExt{} - - pci, err := ghw.PCI() - if err != nil { - return nil, fmt.Errorf("DiscoverSriovDevices(): error getting PCI info: %v", err) - } - - devices := pci.ListDevices() - if len(devices) == 0 { - return nil, fmt.Errorf("DiscoverSriovDevices(): could not retrieve PCI devices") - } - - for _, device := range devices { - devClass, err := strconv.ParseInt(device.Class.ID, 16, 64) - if err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): unable to parse device class, skipping", - "device", device) - continue - } - if devClass != netClass { - // Not network device - continue - } - - // TODO: exclude devices used by host system - - if dputils.IsSriovVF(device.Address) { - continue - } - - driver, err := dputils.GetDriverName(device.Address) - if err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): unable to parse device driver for device, skipping", "device", device) - continue - } - - deviceNames, err := dputils.GetNetNames(device.Address) - if err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): unable to get device names for device, skipping", "device", device) - continue - } - - if len(deviceNames) == 0 { - // no network devices found, skipping device - continue - } - - if !withUnsupported { - if !sriovnetworkv1.IsSupportedModel(device.Vendor.ID, device.Product.ID) { - log.Log.Info("DiscoverSriovDevices(): unsupported device", "device", device) - continue - } - } - - iface := sriovnetworkv1.InterfaceExt{ - PciAddress: device.Address, - Driver: driver, - Vendor: device.Vendor.ID, - DeviceID: device.Product.ID, - } - if mtu := getNetdevMTU(device.Address); mtu > 0 { - iface.Mtu = mtu - } - if name := tryGetInterfaceName(device.Address); name != "" { - iface.Name = name - iface.Mac = getNetDevMac(name) - iface.LinkSpeed = getNetDevLinkSpeed(name) - } - iface.LinkType = getLinkType(iface) - - pfStatus, exist, err := storeManager.LoadPfsStatus(iface.PciAddress) - if err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): failed to load PF status from disk") - } else { - if exist { - iface.ExternallyManaged = pfStatus.ExternallyManaged - } - } - - if dputils.IsSriovPF(device.Address) { - iface.TotalVfs = dputils.GetSriovVFcapacity(device.Address) - iface.NumVfs = dputils.GetVFconfigured(device.Address) - if iface.EswitchMode, err = GetNicSriovMode(device.Address); err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): warning, unable to get device eswitch mode", - "device", device.Address) - } - if dputils.SriovConfigured(device.Address) { - vfs, err := dputils.GetVFList(device.Address) - if err != nil { - log.Log.Error(err, "DiscoverSriovDevices(): unable to parse VFs for device, skipping", - "device", device) - continue - } - for _, vf := range vfs { - instance := getVfInfo(vf, devices) - iface.VFs = append(iface.VFs, instance) - } - } - } - pfList = append(pfList, iface) - } - - return pfList, nil -} - -// SyncNodeState Attempt to update the node state to match the desired state -func SyncNodeState(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToConfig map[string]bool) error { - return ConfigSriovInterfaces(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig) -} - -func ConfigSriovInterfaces(interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error { - if IsKernelLockdownMode(true) && hasMellanoxInterfacesInSpec(ifaceStatuses, interfaces) { - log.Log.Error(nil, "cannot use mellanox devices when in kernel lockdown mode") - return fmt.Errorf("cannot use mellanox devices when in kernel lockdown mode") - } - - // we are already inside chroot, so we initialize the store as running on host - storeManager, err := NewStoreManager(true) - if err != nil { - return fmt.Errorf("SyncNodeState(): error initializing storeManager: %v", err) - } - - for _, ifaceStatus := range ifaceStatuses { - configured := false - for _, iface := range interfaces { - if iface.PciAddress == ifaceStatus.PciAddress { - configured = true - - if skip := pfsToConfig[iface.PciAddress]; skip { - break - } - - if !NeedUpdate(&iface, &ifaceStatus) { - log.Log.V(2).Info("syncNodeState(): no need update interface", "address", iface.PciAddress) - - // Save the PF status to the host - err = storeManager.SaveLastPfAppliedStatus(&iface) - if err != nil { - log.Log.Error(err, "SyncNodeState(): failed to save PF applied config to host") - return err - } - - break - } - if err = configSriovDevice(&iface, &ifaceStatus); err != nil { - log.Log.Error(err, "SyncNodeState(): fail to configure sriov interface. resetting interface.", "address", iface.PciAddress) - if iface.ExternallyManaged { - log.Log.Info("SyncNodeState(): skipping device reset as the nic is marked as externally created") - } else { - if resetErr := resetSriovDevice(ifaceStatus); resetErr != nil { - log.Log.Error(resetErr, "SyncNodeState(): failed to reset on error SR-IOV interface") - } - } - return err - } - - // Save the PF status to the host - err = storeManager.SaveLastPfAppliedStatus(&iface) - if err != nil { - log.Log.Error(err, "SyncNodeState(): failed to save PF applied config to host") - return err - } - break - } - } - if !configured && ifaceStatus.NumVfs > 0 { - if skip := pfsToConfig[ifaceStatus.PciAddress]; skip { - continue - } - - // load the PF info - pfStatus, exist, err := storeManager.LoadPfsStatus(ifaceStatus.PciAddress) - if err != nil { - log.Log.Error(err, "SyncNodeState(): failed to load info about PF status for device", - "address", ifaceStatus.PciAddress) - return err - } - - if !exist { - log.Log.Info("SyncNodeState(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping the device reset", - "pf-name", ifaceStatus.Name, - "address", ifaceStatus.PciAddress) - continue - } - - if pfStatus.ExternallyManaged { - log.Log.Info("SyncNodeState(): PF name with pci address was externally created skipping the device reset", - "pf-name", ifaceStatus.Name, - "address", ifaceStatus.PciAddress) - continue - } else { - err = RemoveUdevRule(ifaceStatus.PciAddress) - if err != nil { - return err - } - } - - if err = resetSriovDevice(ifaceStatus); err != nil { - return err - } - } - } - return nil -} - -// skipConfigVf Use systemd service to configure switchdev mode or BF-2 NICs in OpenShift -func skipConfigVf(ifSpec sriovnetworkv1.Interface, ifStatus sriovnetworkv1.InterfaceExt) (bool, error) { - if ifSpec.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - log.Log.V(2).Info("skipConfigVf(): skip config VF for switchdev device") - return true, nil - } - - // NVIDIA BlueField 2 and BlueField3 in OpenShift - if ClusterType == ClusterTypeOpenshift && ifStatus.Vendor == VendorMellanox && (ifStatus.DeviceID == DeviceBF2 || ifStatus.DeviceID == DeviceBF3) { - // TODO: remove this when switch to the systemd configuration support. - mode, err := mellanoxBlueFieldMode(ifStatus.PciAddress) - if err != nil { - return false, fmt.Errorf("failed to read Mellanox Bluefield card mode for %s,%v", ifStatus.PciAddress, err) - } - - if mode == bluefieldConnectXMode { - return false, nil - } - - log.Log.V(2).Info("skipConfigVf(): skip config VF for Bluefiled card on DPU mode") - return true, nil - } - - return false, nil -} - -// GetPfsToSkip return a map of devices pci addresses to should be configured via systemd instead if the legacy mode -// we skip devices in switchdev mode and Bluefield card in ConnectX mode -func GetPfsToSkip(ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]bool, error) { - pfsToSkip := map[string]bool{} - for _, ifaceStatus := range ns.Status.Interfaces { - for _, iface := range ns.Spec.Interfaces { - if iface.PciAddress == ifaceStatus.PciAddress { - skip, err := skipConfigVf(iface, ifaceStatus) - if err != nil { - log.Log.Error(err, "GetPfsToSkip(): fail to check for skip VFs", "device", iface.PciAddress) - return pfsToSkip, err - } - pfsToSkip[iface.PciAddress] = skip - break - } - } - } - - return pfsToSkip, nil -} - -func NeedUpdate(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) bool { - if iface.Mtu > 0 { - mtu := iface.Mtu - if mtu != ifaceStatus.Mtu { - log.Log.V(2).Info("NeedUpdate(): MTU needs update", "desired", mtu, "current", ifaceStatus.Mtu) - return true - } - } - - if iface.NumVfs != ifaceStatus.NumVfs { - log.Log.V(2).Info("NeedUpdate(): NumVfs needs update", "desired", iface.NumVfs, "current", ifaceStatus.NumVfs) - return true - } - if iface.NumVfs > 0 { - for _, vf := range ifaceStatus.VFs { - ingroup := false - for _, group := range iface.VfGroups { - if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) { - ingroup = true - if group.DeviceType != constants.DeviceTypeNetDevice { - if group.DeviceType != vf.Driver { - log.Log.V(2).Info("NeedUpdate(): Driver needs update", - "desired", group.DeviceType, "current", vf.Driver) - return true - } - } else { - if sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) { - log.Log.V(2).Info("NeedUpdate(): Driver needs update", - "desired", group.DeviceType, "current", vf.Driver) - return true - } - if vf.Mtu != 0 && group.Mtu != 0 && vf.Mtu != group.Mtu { - log.Log.V(2).Info("NeedUpdate(): VF MTU needs update", - "vf", vf.VfID, "desired", group.Mtu, "current", vf.Mtu) - return true - } - - // this is needed to be sure the admin mac address is configured as expected - if iface.ExternallyManaged { - log.Log.V(2).Info("NeedUpdate(): need to update the device as it's externally manage", - "device", ifaceStatus.PciAddress) - return true - } - } - break - } - } - if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, DpdkDrivers) { - // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver. - return true - } - } - } - return false -} - -func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) error { - log.Log.V(2).Info("configSriovDevice(): configure sriov device", - "device", iface.PciAddress, "config", iface) - var err error - if iface.NumVfs > ifaceStatus.TotalVfs { - err := fmt.Errorf("cannot config SRIOV device: NumVfs (%d) is larger than TotalVfs (%d)", iface.NumVfs, ifaceStatus.TotalVfs) - log.Log.Error(err, "configSriovDevice(): fail to set NumVfs for device", "device", iface.PciAddress) - return err - } - // set numVFs - if iface.NumVfs != ifaceStatus.NumVfs { - if iface.ExternallyManaged { - if iface.NumVfs > ifaceStatus.NumVfs { - errMsg := fmt.Sprintf("configSriovDevice(): number of request virtual functions %d is not equal to configured virtual functions %d but the policy is configured as ExternallyManaged for device %s", iface.NumVfs, ifaceStatus.NumVfs, iface.PciAddress) - log.Log.Error(nil, errMsg) - return fmt.Errorf(errMsg) - } - } else { - // create the udev rule to disable all the vfs from network manager as this vfs are managed by the operator - err = AddUdevRule(iface.PciAddress) - if err != nil { - return err - } - - err = setSriovNumVfs(iface.PciAddress, iface.NumVfs) - if err != nil { - log.Log.Error(err, "configSriovDevice(): fail to set NumVfs for device", "device", iface.PciAddress) - errRemove := RemoveUdevRule(iface.PciAddress) - if errRemove != nil { - log.Log.Error(errRemove, "configSriovDevice(): fail to remove udev rule", "device", iface.PciAddress) - } - return err - } - } - } - // set PF mtu - if iface.Mtu > 0 && iface.Mtu > ifaceStatus.Mtu { - err = setNetdevMTU(iface.PciAddress, iface.Mtu) - if err != nil { - log.Log.Error(err, "configSriovDevice(): fail to set mtu for PF", "device", iface.PciAddress) - return err - } - } - // Config VFs - if iface.NumVfs > 0 { - vfAddrs, err := dputils.GetVFList(iface.PciAddress) - if err != nil { - log.Log.Error(err, "configSriovDevice(): unable to parse VFs for device", "device", iface.PciAddress) - } - pfLink, err := netlink.LinkByName(iface.Name) - if err != nil { - log.Log.Error(err, "configSriovDevice(): unable to get PF link for device", "device", iface) - return err - } - - for _, addr := range vfAddrs { - var group *sriovnetworkv1.VfGroup - - vfID, err := dputils.GetVFID(addr) - if err != nil { - log.Log.Error(err, "configSriovDevice(): unable to get VF id", "device", iface.PciAddress) - return err - } - - for i := range iface.VfGroups { - if sriovnetworkv1.IndexInRange(vfID, iface.VfGroups[i].VfRange) { - group = &iface.VfGroups[i] - break - } - } - - // VF group not found. - if group == nil { - continue - } - - // only set GUID and MAC for VF with default driver - // for userspace drivers like vfio we configure the vf mac using the kernel nic mac address - // before we switch to the userspace driver - if yes, d := hasDriver(addr); yes && !sriovnetworkv1.StringInArray(d, DpdkDrivers) { - // LinkType is an optional field. Let's fallback to current link type - // if nothing is specified in the SriovNodePolicy - linkType := iface.LinkType - if linkType == "" { - linkType = ifaceStatus.LinkType - } - if strings.EqualFold(linkType, constants.LinkTypeIB) { - if err = setVfGUID(addr, pfLink); err != nil { - return err - } - } else { - vfLink, err := vfIsReady(addr) - if err != nil { - log.Log.Error(err, "configSriovDevice(): VF link is not ready", "address", addr) - err = RebindVfToDefaultDriver(addr) - if err != nil { - log.Log.Error(err, "configSriovDevice(): failed to rebind VF", "address", addr) - return err - } - - // Try to check the VF status again - vfLink, err = vfIsReady(addr) - if err != nil { - log.Log.Error(err, "configSriovDevice(): VF link is not ready", "address", addr) - return err - } - } - if err = setVfAdminMac(addr, pfLink, vfLink); err != nil { - log.Log.Error(err, "configSriovDevice(): fail to configure VF admin mac", "device", addr) - return err - } - } - } - - if err = unbindDriverIfNeeded(addr, group.IsRdma); err != nil { - return err - } - - if !sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) { - if err := BindDefaultDriver(addr); err != nil { - log.Log.Error(err, "configSriovDevice(): fail to bind default driver for device", "device", addr) - return err - } - // only set MTU for VF with default driver - if group.Mtu > 0 { - if err := setNetdevMTU(addr, group.Mtu); err != nil { - log.Log.Error(err, "configSriovDevice(): fail to set mtu for VF", "address", addr) - return err - } - } - } else { - if err := BindDpdkDriver(addr, group.DeviceType); err != nil { - log.Log.Error(err, "configSriovDevice(): fail to bind driver for device", - "driver", group.DeviceType, "device", addr) - return err - } - } - } - } - // Set PF link up - pfLink, err := netlink.LinkByName(ifaceStatus.Name) - if err != nil { - return err - } - if pfLink.Attrs().OperState != netlink.OperUp { - err = netlink.LinkSetUp(pfLink) - if err != nil { - return err - } - } - return nil -} - -func setSriovNumVfs(pciAddr string, numVfs int) error { - log.Log.V(2).Info("setSriovNumVfs(): set NumVfs", "device", pciAddr, "numVfs", numVfs) - numVfsFilePath := filepath.Join(sysBusPciDevices, pciAddr, numVfsFile) - bs := []byte(strconv.Itoa(numVfs)) - err := os.WriteFile(numVfsFilePath, []byte("0"), os.ModeAppend) - if err != nil { - log.Log.Error(err, "setSriovNumVfs(): fail to reset NumVfs file", "path", numVfsFilePath) - return err - } - err = os.WriteFile(numVfsFilePath, bs, os.ModeAppend) - if err != nil { - log.Log.Error(err, "setSriovNumVfs(): fail to set NumVfs file", "path", numVfsFilePath) - return err - } - return nil -} - -func setNetdevMTU(pciAddr string, mtu int) error { - log.Log.V(2).Info("setNetdevMTU(): set MTU", "device", pciAddr, "mtu", mtu) - if mtu <= 0 { - log.Log.V(2).Info("setNetdevMTU(): refusing to set MTU", "mtu", mtu) - return nil - } - b := backoff.NewConstantBackOff(1 * time.Second) - err := backoff.Retry(func() error { - ifaceName, err := dputils.GetNetNames(pciAddr) - if err != nil { - log.Log.Error(err, "setNetdevMTU(): fail to get interface name", "device", pciAddr) - return err - } - if len(ifaceName) < 1 { - return fmt.Errorf("setNetdevMTU(): interface name is empty") - } - mtuFile := "net/" + ifaceName[0] + "/mtu" - mtuFilePath := filepath.Join(sysBusPciDevices, pciAddr, mtuFile) - return os.WriteFile(mtuFilePath, []byte(strconv.Itoa(mtu)), os.ModeAppend) - }, backoff.WithMaxRetries(b, 10)) - if err != nil { - log.Log.Error(err, "setNetdevMTU(): fail to write mtu file after retrying") - return err - } - return nil -} - -func tryGetInterfaceName(pciAddr string) string { - names, err := dputils.GetNetNames(pciAddr) - if err != nil || len(names) < 1 { - return "" - } - netDevName := names[0] - - // Switchdev PF and their VFs representors are existing under the same PCI address since kernel 5.8 - // if device is switchdev then return PF name - for _, name := range names { - if !isSwitchdev(name) { - continue - } - // Try to get the phys port name, if not exists then fallback to check without it - // phys_port_name should be in formant p e.g p0,p1,p2 ...etc. - if physPortName, err := GetPhysPortName(name); err == nil { - if !pfPhysPortNameRe.MatchString(physPortName) { - continue - } - } - return name - } - - log.Log.V(2).Info("tryGetInterfaceName()", "name", netDevName) - return netDevName -} - -func getNetdevMTU(pciAddr string) int { - log.Log.V(2).Info("getNetdevMTU(): get MTU", "device", pciAddr) - ifaceName := tryGetInterfaceName(pciAddr) - if ifaceName == "" { - return 0 - } - mtuFile := "net/" + ifaceName + "/mtu" - mtuFilePath := filepath.Join(sysBusPciDevices, pciAddr, mtuFile) - data, err := os.ReadFile(mtuFilePath) - if err != nil { - log.Log.Error(err, "getNetdevMTU(): fail to read mtu file", "path", mtuFilePath) - return 0 - } - mtu, err := strconv.Atoi(strings.TrimSpace(string(data))) - if err != nil { - log.Log.Error(err, "getNetdevMTU(): fail to convert mtu to int", "raw-mtu", strings.TrimSpace(string(data))) - return 0 - } - return mtu -} - -func getNetDevMac(ifaceName string) string { - log.Log.V(2).Info("getNetDevMac(): get Mac", "device", ifaceName) - macFilePath := filepath.Join(sysClassNet, ifaceName, "address") - data, err := os.ReadFile(macFilePath) - if err != nil { - log.Log.Error(err, "getNetDevMac(): fail to read Mac file", "path", macFilePath) - return "" - } - - return strings.TrimSpace(string(data)) -} - -func getNetDevLinkSpeed(ifaceName string) string { - log.Log.V(2).Info("getNetDevLinkSpeed(): get LinkSpeed", "device", ifaceName) - speedFilePath := filepath.Join(sysClassNet, ifaceName, "speed") - data, err := os.ReadFile(speedFilePath) - if err != nil { - log.Log.Error(err, "getNetDevLinkSpeed(): fail to read Link Speed file", "path", speedFilePath) - return "" - } - - return fmt.Sprintf("%s Mb/s", strings.TrimSpace(string(data))) +//go:generate ../../bin/mockgen -destination mock/mock_utils.go -source utils.go +type CmdInterface interface { + Chroot(string) (func() error, error) + RunCommand(string, ...string) (string, string, error) } -func resetSriovDevice(ifaceStatus sriovnetworkv1.InterfaceExt) error { - log.Log.V(2).Info("resetSriovDevice(): reset SRIOV device", "address", ifaceStatus.PciAddress) - if err := setSriovNumVfs(ifaceStatus.PciAddress, 0); err != nil { - return err - } - if ifaceStatus.LinkType == constants.LinkTypeETH { - var mtu int - is := InitialState.GetInterfaceStateByPciAddress(ifaceStatus.PciAddress) - if is != nil { - mtu = is.Mtu - } else { - mtu = 1500 - } - log.Log.V(2).Info("resetSriovDevice(): reset mtu", "value", mtu) - if err := setNetdevMTU(ifaceStatus.PciAddress, mtu); err != nil { - return err - } - } else if ifaceStatus.LinkType == constants.LinkTypeIB { - if err := setNetdevMTU(ifaceStatus.PciAddress, 2048); err != nil { - return err - } - } - return nil +type utilsHelper struct { } -func getVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { - driver, err := dputils.GetDriverName(pciAddr) - if err != nil { - log.Log.Error(err, "getVfInfo(): unable to parse device driver", "device", pciAddr) - } - id, err := dputils.GetVFID(pciAddr) - if err != nil { - log.Log.Error(err, "getVfInfo(): unable to get VF index", "device", pciAddr) - } - vf := sriovnetworkv1.VirtualFunction{ - PciAddress: pciAddr, - Driver: driver, - VfID: id, - } - - if mtu := getNetdevMTU(pciAddr); mtu > 0 { - vf.Mtu = mtu - } - if name := tryGetInterfaceName(pciAddr); name != "" { - vf.Name = name - vf.Mac = getNetDevMac(name) - } - - for _, device := range devices { - if pciAddr == device.Address { - vf.Vendor = device.Vendor.ID - vf.DeviceID = device.Product.ID - break - } - continue - } - return vf +func New() CmdInterface { + return &utilsHelper{} } -func Chroot(path string) (func() error, error) { +func (u *utilsHelper) Chroot(path string) (func() error, error) { root, err := os.Open("/") if err != nil { return nil, err @@ -746,101 +43,33 @@ func Chroot(path string) (func() error, error) { root.Close() return nil, err } + vars.InChroot = true return func() error { defer root.Close() if err := root.Chdir(); err != nil { return err } + vars.InChroot = false return syscall.Chroot(".") }, nil } -func vfIsReady(pciAddr string) (netlink.Link, error) { - log.Log.Info("vfIsReady()", "device", pciAddr) - var err error - var vfLink netlink.Link - err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - vfName := tryGetInterfaceName(pciAddr) - vfLink, err = netlink.LinkByName(vfName) - if err != nil { - log.Log.Error(err, "vfIsReady(): unable to get VF link", "device", pciAddr) - } - return err == nil, nil - }) - if err != nil { - return vfLink, err - } - return vfLink, nil -} - -func setVfAdminMac(vfAddr string, pfLink, vfLink netlink.Link) error { - log.Log.Info("setVfAdminMac()", "vf", vfAddr) - - vfID, err := dputils.GetVFID(vfAddr) - if err != nil { - log.Log.Error(err, "setVfAdminMac(): unable to get VF id", "address", vfAddr) - return err - } - - if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { - return err - } - - return nil -} - -func unbindDriverIfNeeded(vfAddr string, isRdma bool) error { - if isRdma { - log.Log.Info("unbindDriverIfNeeded(): unbind driver", "device", vfAddr) - if err := Unbind(vfAddr); err != nil { - return err - } - } - return nil -} - -func getLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string { - log.Log.V(2).Info("getLinkType()", "device", ifaceStatus.PciAddress) - if ifaceStatus.Name != "" { - link, err := netlink.LinkByName(ifaceStatus.Name) - if err != nil { - log.Log.Error(err, "getLinkType(): failed to get link", "device", ifaceStatus.Name) - return "" - } - linkType := link.Attrs().EncapType - if linkType == "ether" { - return constants.LinkTypeETH - } else if linkType == "infiniband" { - return constants.LinkTypeIB - } - } - - return "" -} +// RunCommand runs a command +func (u *utilsHelper) RunCommand(command string, args ...string) (string, string, error) { + log.Log.Info("RunCommand()", "command", command, "args", args) + var stdout, stderr bytes.Buffer -func setVfGUID(vfAddr string, pfLink netlink.Link) error { - log.Log.Info("setVfGuid()", "vf", vfAddr) - vfID, err := dputils.GetVFID(vfAddr) - if err != nil { - log.Log.Error(err, "setVfGuid(): unable to get VF id", "address", vfAddr) - return err - } - guid := generateRandomGUID() - if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { - return err - } - if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { - return err - } - if err = Unbind(vfAddr); err != nil { - return err - } + cmd := exec.Command(command, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr - return nil + err := cmd.Run() + log.Log.V(2).Info("RunCommand()", "output", stdout.String(), "error", err) + return stdout.String(), stderr.String(), err } -func generateRandomGUID() net.HardwareAddr { +func GenerateRandomGUID() net.HardwareAddr { guid := make(net.HardwareAddr, 8) // First field is 0x01 - xfe to avoid all zero and all F invalid guids @@ -853,82 +82,6 @@ func generateRandomGUID() net.HardwareAddr { return guid } -func GetNicSriovMode(pciAddress string) (string, error) { - log.Log.V(2).Info("GetNicSriovMode()", "device", pciAddress) - - devLink, err := netlink.DevLinkGetDeviceByName("pci", pciAddress) - if err != nil { - if errors.Is(err, syscall.ENODEV) { - // the device doesn't support devlink - return "", nil - } - return "", err - } - - return devLink.Attrs.Eswitch.Mode, nil -} - -func GetPhysSwitchID(name string) (string, error) { - swIDFile := filepath.Join(sysClassNet, name, "phys_switch_id") - physSwitchID, err := os.ReadFile(swIDFile) - if err != nil { - return "", err - } - if physSwitchID != nil { - return strings.TrimSpace(string(physSwitchID)), nil - } - return "", nil -} - -func GetPhysPortName(name string) (string, error) { - devicePortNameFile := filepath.Join(sysClassNet, name, "phys_port_name") - physPortName, err := os.ReadFile(devicePortNameFile) - if err != nil { - return "", err - } - if physPortName != nil { - return strings.TrimSpace(string(physPortName)), nil - } - return "", nil -} - -func isSwitchdev(name string) bool { - switchID, err := GetPhysSwitchID(name) - if err != nil || switchID == "" { - return false - } - - return true -} - -// IsKernelLockdownMode returns true when kernel lockdown mode is enabled -func IsKernelLockdownMode(chroot bool) bool { - path := "/sys/kernel/security/lockdown" - if !chroot { - path = "/host" + path - } - out, err := RunCommand("cat", path) - log.Log.V(2).Info("IsKernelLockdownMode()", "output", out, "error", err) - if err != nil { - return false - } - return strings.Contains(out, "[integrity]") || strings.Contains(out, "[confidentiality]") -} - -// RunCommand runs a command -func RunCommand(command string, args ...string) (string, error) { - log.Log.Info("RunCommand()", "command", command, "args", args) - var stdout, stderr bytes.Buffer - - cmd := exec.Command(command, args...) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - log.Log.V(2).Info("RunCommand()", "output", stdout.String(), "error", err) - return stdout.String(), err -} - func HashConfigMap(cm *corev1.ConfigMap) string { var keys []string for k := range cm.Data { @@ -945,97 +98,25 @@ func HashConfigMap(cm *corev1.ConfigMap) string { return hex.EncodeToString(hashed) } -func hasMellanoxInterfacesInSpec(ifaceStatuses sriovnetworkv1.InterfaceExts, ifaceSpecs sriovnetworkv1.Interfaces) bool { - for _, ifaceStatus := range ifaceStatuses { - if ifaceStatus.Vendor == VendorMellanox { - for _, iface := range ifaceSpecs { - if iface.PciAddress == ifaceStatus.PciAddress { - log.Log.V(2).Info("hasMellanoxInterfacesInSpec(): Mellanox device specified in SriovNetworkNodeState spec", - "name", ifaceStatus.Name, - "address", ifaceStatus.PciAddress) - return true - } - } +func IsCommandNotFound(err error) bool { + if exitErr, ok := err.(*exec.ExitError); ok { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 { + return true } } return false } -// Workaround function to handle a case where the vf default driver is stuck and not able to create the vf kernel interface. -// This function unbind the VF from the default driver and try to bind it again -// bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2045087 -func RebindVfToDefaultDriver(vfAddr string) error { - log.Log.Info("RebindVfToDefaultDriver()", "vf", vfAddr) - if err := Unbind(vfAddr); err != nil { - return err - } - if err := BindDefaultDriver(vfAddr); err != nil { - log.Log.Error(err, "RebindVfToDefaultDriver(): fail to bind default driver", "device", vfAddr) - return err - } - - log.Log.Info("RebindVfToDefaultDriver(): workaround implemented", "vf", vfAddr) - return nil -} - -func PrepareNMUdevRule(supportedVfIds []string) error { - log.Log.V(2).Info("PrepareNMUdevRule()") - dirPath := path.Join(FilesystemRoot, "/host/etc/udev/rules.d") - filePath := path.Join(dirPath, "10-nm-unmanaged.rules") - - // remove the old unmanaged rules file - if _, err := os.Stat(filePath); err == nil { - err = os.Remove(filePath) - if err != nil { - log.Log.Error(err, "failed to remove the network manager global unmanaged rule", - "path", filePath) - } - } - - // create the pf finder script for udev rules - var stdout, stderr bytes.Buffer - cmd := exec.Command("/bin/bash", path.Join(FilesystemRoot, udevDisableNM)) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - log.Log.Error(err, "PrepareNMUdevRule(): failed to prepare nmUdevRule", "stderr", stderr.String()) - return err - } - log.Log.V(2).Info("PrepareNMUdevRule()", "stdout", stdout.String()) - - //save the device list to use for udev rules - SupportedVfIds = supportedVfIds - return nil -} - -func AddUdevRule(pfPciAddress string) error { - log.Log.V(2).Info("AddUdevRule()", "device", pfPciAddress) - pathFile := udevRulesFolder - udevRuleContent := fmt.Sprintf(nmUdevRule, strings.Join(SupportedVfIds, "|"), pfPciAddress) - - err := os.MkdirAll(pathFile, os.ModePerm) - if err != nil && !os.IsExist(err) { - log.Log.Error(err, "AddUdevRule(): failed to create dir", "path", pathFile) - return err - } - - filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) - // if the file does not exist or if oldContent != newContent - // write to file and create it if it doesn't exist - err = os.WriteFile(filePath, []byte(udevRuleContent), 0666) - if err != nil { - log.Log.Error(err, "AddUdevRule(): fail to write file", "path", filePath) - return err +func GetHostExtension() string { + if vars.InChroot { + return vars.FilesystemRoot } - return nil + return filepath.Join(vars.FilesystemRoot, consts.Host) } -func RemoveUdevRule(pfPciAddress string) error { - pathFile := udevRulesFolder - filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) - err := os.Remove(filePath) - if err != nil && !os.IsNotExist(err) { - return err +func GetChrootExtension() string { + if vars.InChroot { + return vars.FilesystemRoot } - return nil + return fmt.Sprintf("chroot %s%s", vars.FilesystemRoot, consts.Host) } diff --git a/pkg/utils/utils_mlx.go b/pkg/utils/utils_mlx.go deleted file mode 100644 index ace265c79..000000000 --- a/pkg/utils/utils_mlx.go +++ /dev/null @@ -1,118 +0,0 @@ -package utils - -import ( - "fmt" - "regexp" - "strings" - - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// BlueField mode representation -type BlueFieldMode int - -const ( - bluefieldDpu BlueFieldMode = iota - bluefieldConnectXMode - - internalCPUPageSupplier = "INTERNAL_CPU_PAGE_SUPPLIER" - internalCPUEswitchManager = "INTERNAL_CPU_ESWITCH_MANAGER" - internalCPUIbVporto = "INTERNAL_CPU_IB_VPORT0" - internalCPUOffloadEngine = "INTERNAL_CPU_OFFLOAD_ENGINE" - internalCPUModel = "INTERNAL_CPU_MODEL" - - ecpf = "ECPF" - extHostPf = "EXT_HOST_PF" - embeddedCPU = "EMBEDDED_CPU" - - disabled = "DISABLED" - enabled = "ENABLED" -) - -func MstConfigReadData(pciAddress string) (string, error) { - log.Log.Info("MstConfigReadData()", "device", pciAddress) - args := []string{"-e", "-d", pciAddress, "q"} - out, err := RunCommand("mstconfig", args...) - return out, err -} - -func ParseMstconfigOutput(mstOutput string, attributes []string) (fwCurrent, fwNext map[string]string) { - log.Log.Info("ParseMstconfigOutput()", "attributes", attributes) - fwCurrent = map[string]string{} - fwNext = map[string]string{} - formatRegex := regexp.MustCompile(`(?P\w+)\s+(?P\S+)\s+(?P\S+)\s+(?P\S+)`) - mstOutputLines := strings.Split(mstOutput, "\n") - for _, attr := range attributes { - for _, line := range mstOutputLines { - if strings.Contains(line, attr) { - regexResult := formatRegex.FindStringSubmatch(line) - fwCurrent[attr] = regexResult[3] - fwNext[attr] = regexResult[4] - break - } - } - } - return -} - -func mellanoxBlueFieldMode(PciAddress string) (BlueFieldMode, error) { - log.Log.V(2).Info("MellanoxBlueFieldMode(): checking mode for device", "device", PciAddress) - out, err := MstConfigReadData(PciAddress) - if err != nil { - log.Log.Error(err, "MellanoxBlueFieldMode(): failed to get mlx nic fw data") - return -1, fmt.Errorf("failed to get mlx nic fw data %w", err) - } - - attrs := []string{internalCPUPageSupplier, - internalCPUEswitchManager, - internalCPUIbVporto, - internalCPUOffloadEngine, - internalCPUModel} - mstCurrentData, _ := ParseMstconfigOutput(out, attrs) - - internalCPUPageSupplierstatus, exist := mstCurrentData[internalCPUPageSupplier] - if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier) - } - - internalCPUEswitchManagerStatus, exist := mstCurrentData[internalCPUEswitchManager] - if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager) - } - - internalCPUIbVportoStatus, exist := mstCurrentData[internalCPUIbVporto] - if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto) - } - - internalCPUOffloadEngineStatus, exist := mstCurrentData[internalCPUOffloadEngine] - if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine) - } - - internalCPUModelStatus, exist := mstCurrentData[internalCPUModel] - if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel) - } - - // check for DPU - if strings.Contains(internalCPUPageSupplierstatus, ecpf) && - strings.Contains(internalCPUEswitchManagerStatus, ecpf) && - strings.Contains(internalCPUIbVportoStatus, ecpf) && - strings.Contains(internalCPUOffloadEngineStatus, enabled) && - strings.Contains(internalCPUModelStatus, embeddedCPU) { - log.Log.V(2).Info("MellanoxBlueFieldMode(): device in DPU mode", "device", PciAddress) - return bluefieldDpu, nil - } else if strings.Contains(internalCPUPageSupplierstatus, extHostPf) && - strings.Contains(internalCPUEswitchManagerStatus, extHostPf) && - strings.Contains(internalCPUIbVportoStatus, extHostPf) && - strings.Contains(internalCPUOffloadEngineStatus, disabled) && - strings.Contains(internalCPUModelStatus, embeddedCPU) { - log.Log.V(2).Info("MellanoxBlueFieldMode(): device in ConnectX mode", "device", PciAddress) - return bluefieldConnectXMode, nil - } - - log.Log.Error(err, "MellanoxBlueFieldMode(): unknown device status", - "device", PciAddress, "mstconfig-output", out) - return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress) -} diff --git a/pkg/vars/vars.go b/pkg/vars/vars.go new file mode 100644 index 000000000..a34cab8af --- /dev/null +++ b/pkg/vars/vars.go @@ -0,0 +1,82 @@ +package vars + +import ( + "os" + "regexp" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" +) + +var ( + // ClusterType used by the operator to specify the platform it's running on + // supported values [kubernetes,openshift] + ClusterType string + + // DevMode controls the developer mode in the operator + // developer mode allows the operator to use un-supported network devices + DevMode bool + + // EnableAdmissionController allows the user to disable the operator webhooks + EnableAdmissionController bool + + // NodeName initialize and used by the config-daemon to identify the node it's running on + NodeName = "" + + // Destdir destination directory for the checkPoint file on the host + Destdir string + + // PlatformType specify the current platform the operator is running on + PlatformType = consts.Baremetal + // PlatformsMap contains supported platforms for virtual VF + PlatformsMap = map[string]consts.PlatformTypes{ + "openstack": consts.VirtualOpenStack, + } + + // SupportedVfIds list of supported virtual functions IDs + // loaded on daemon initialization by reading the supported-nics configmap + SupportedVfIds []string + + // DpdkDrivers supported DPDK drivers for virtual functions + DpdkDrivers = []string{"igb_uio", "vfio-pci", "uio_pci_generic"} + + // InChroot global variable to mark that the config-daemon code is inside chroot on the host file system + InChroot = false + + // UsingSystemdMode global variable to mark the config-daemon is running on systemd mode + UsingSystemdMode = false + + // FilesystemRoot used by test to mock interactions with filesystem + FilesystemRoot = "" + + //Cluster variables + Config *rest.Config = nil + Scheme *runtime.Scheme = nil + + // PfPhysPortNameRe regex to find switchdev devices on the host + PfPhysPortNameRe = regexp.MustCompile(`p\d+`) +) + +func init() { + ClusterType = os.Getenv("CLUSTER_TYPE") + + DevMode = false + mode := os.Getenv("DEV_MODE") + if mode == "TRUE" { + DevMode = true + } + + Destdir = "/host/tmp" + destdir := os.Getenv("DEST_DIR") + if destdir != "" { + Destdir = destdir + } + + EnableAdmissionController = false + enableAdmissionController := os.Getenv("ADMISSION_CONTROLLERS_ENABLED") + if enableAdmissionController == "True" { + EnableAdmissionController = true + } +} diff --git a/pkg/vendors/mellanox/mellanox.go b/pkg/vendors/mellanox/mellanox.go new file mode 100644 index 000000000..c6631ed28 --- /dev/null +++ b/pkg/vendors/mellanox/mellanox.go @@ -0,0 +1,419 @@ +package mlxutils + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +// BlueField mode representation +type BlueFieldMode int + +const ( + BluefieldDpu BlueFieldMode = iota + BluefieldConnectXMode + + internalCPUPageSupplier = "INTERNAL_CPU_PAGE_SUPPLIER" + internalCPUEswitchManager = "INTERNAL_CPU_ESWITCH_MANAGER" + internalCPUIbVporto = "INTERNAL_CPU_IB_VPORT0" + internalCPUOffloadEngine = "INTERNAL_CPU_OFFLOAD_ENGINE" + internalCPUModel = "INTERNAL_CPU_MODEL" + + ecpf = "ECPF" + extHostPf = "EXT_HOST_PF" + embeddedCPU = "EMBEDDED_CPU" + + disabled = "DISABLED" + enabled = "ENABLED" + + VendorMellanox = "15b3" + DeviceBF2 = "a2d6" + DeviceBF3 = "a2dc" + + PreconfiguredLinkType = "Preconfigured" + UknownLinkType = "Uknown" + TotalVfs = "NUM_OF_VFS" + EnableSriov = "SRIOV_EN" + LinkTypeP1 = "LINK_TYPE_P1" + LinkTypeP2 = "LINK_TYPE_P2" + MellanoxVendorID = "15b3" +) + +type MlxNic struct { + EnableSriov bool + TotalVfs int + LinkTypeP1 string + LinkTypeP2 string +} + +//go:generate ../../../bin/mockgen -destination mock/mock_mellanox.go -source mellanox.go +type MellanoxInterface interface { + MstConfigReadData(string) (string, string, error) + GetMellanoxBlueFieldMode(string) (BlueFieldMode, error) + GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error) + + MlxConfigFW(attributesToChange map[string]MlxNic) error +} + +type mellanoxHelper struct { + utils utils.CmdInterface +} + +func New(utilsHelper utils.CmdInterface) MellanoxInterface { + return &mellanoxHelper{ + utils: utilsHelper, + } +} + +func (m *mellanoxHelper) MstConfigReadData(pciAddress string) (string, string, error) { + log.Log.Info("MstConfigReadData()", "device", pciAddress) + args := []string{"-e", "-d", pciAddress, "q"} + stdout, stderr, err := m.utils.RunCommand("mstconfig", args...) + return stdout, stderr, err +} + +func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldMode, error) { + log.Log.V(2).Info("MellanoxBlueFieldMode(): checking mode for device", "device", PciAddress) + stdout, stderr, err := m.MstConfigReadData(PciAddress) + if err != nil { + log.Log.Error(err, "MellanoxBlueFieldMode(): failed to get mlx nic fw data", "stderr", stderr) + return -1, fmt.Errorf("failed to get mlx nic fw data %w", err) + } + + attrs := []string{internalCPUPageSupplier, + internalCPUEswitchManager, + internalCPUIbVporto, + internalCPUOffloadEngine, + internalCPUModel} + mstCurrentData, _ := ParseMstconfigOutput(stdout, attrs) + + internalCPUPageSupplierstatus, exist := mstCurrentData[internalCPUPageSupplier] + if !exist { + return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier) + } + + internalCPUEswitchManagerStatus, exist := mstCurrentData[internalCPUEswitchManager] + if !exist { + return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager) + } + + internalCPUIbVportoStatus, exist := mstCurrentData[internalCPUIbVporto] + if !exist { + return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto) + } + + internalCPUOffloadEngineStatus, exist := mstCurrentData[internalCPUOffloadEngine] + if !exist { + return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine) + } + + internalCPUModelStatus, exist := mstCurrentData[internalCPUModel] + if !exist { + return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel) + } + + // check for DPU + if strings.Contains(internalCPUPageSupplierstatus, ecpf) && + strings.Contains(internalCPUEswitchManagerStatus, ecpf) && + strings.Contains(internalCPUIbVportoStatus, ecpf) && + strings.Contains(internalCPUOffloadEngineStatus, enabled) && + strings.Contains(internalCPUModelStatus, embeddedCPU) { + log.Log.V(2).Info("MellanoxBlueFieldMode(): device in DPU mode", "device", PciAddress) + return BluefieldDpu, nil + } else if strings.Contains(internalCPUPageSupplierstatus, extHostPf) && + strings.Contains(internalCPUEswitchManagerStatus, extHostPf) && + strings.Contains(internalCPUIbVportoStatus, extHostPf) && + strings.Contains(internalCPUOffloadEngineStatus, disabled) && + strings.Contains(internalCPUModelStatus, embeddedCPU) { + log.Log.V(2).Info("MellanoxBlueFieldMode(): device in ConnectX mode", "device", PciAddress) + return BluefieldConnectXMode, nil + } + + log.Log.Error(err, "MellanoxBlueFieldMode(): unknown device status", + "device", PciAddress, "mstconfig-output", stdout) + return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress) +} + +func (m *mellanoxHelper) MlxConfigFW(attributesToChange map[string]MlxNic) error { + log.Log.Info("mellanox-plugin configFW()") + for pciAddr, fwArgs := range attributesToChange { + cmdArgs := []string{"-d", pciAddr, "-y", "set"} + if fwArgs.EnableSriov { + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=True", EnableSriov)) + } else if fwArgs.TotalVfs == 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=False", EnableSriov)) + } + if fwArgs.TotalVfs > -1 { + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%d", TotalVfs, fwArgs.TotalVfs)) + } + if len(fwArgs.LinkTypeP1) > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP1, fwArgs.LinkTypeP1)) + } + if len(fwArgs.LinkTypeP2) > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP2, fwArgs.LinkTypeP2)) + } + + log.Log.V(2).Info("mellanox-plugin: configFW()", "cmd-args", cmdArgs) + if len(cmdArgs) <= 4 { + continue + } + _, strerr, err := m.utils.RunCommand("mstconfig", cmdArgs...) + if err != nil { + log.Log.Error(err, "mellanox-plugin configFW(): failed", "stderr", strerr) + return err + } + } + return nil +} + +func (m *mellanoxHelper) GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error) { + log.Log.Info("mellanox-plugin getMlnxNicFwData()", "device", pciAddress) + attrs := []string{TotalVfs, EnableSriov, LinkTypeP1, LinkTypeP2} + + out, stderr, err := m.MstConfigReadData(pciAddress) + if err != nil { + log.Log.Error(err, "mellanox-plugin getMlnxNicFwData(): failed", "stderr", stderr) + return + } + mstCurrentData, mstNextData := ParseMstconfigOutput(out, attrs) + current, err = mlnxNicFromMap(mstCurrentData) + if err != nil { + log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for current mstconfig data failed") + return + } + next, err = mlnxNicFromMap(mstNextData) + if err != nil { + log.Log.Error(err, "mellanox-plugin mlnxNicFromMap() for next mstconfig data failed") + } + return +} + +func ParseMstconfigOutput(mstOutput string, attributes []string) (fwCurrent, fwNext map[string]string) { + log.Log.Info("ParseMstconfigOutput()", "attributes", attributes) + fwCurrent = map[string]string{} + fwNext = map[string]string{} + formatRegex := regexp.MustCompile(`(?P\w+)\s+(?P\S+)\s+(?P\S+)\s+(?P\S+)`) + mstOutputLines := strings.Split(mstOutput, "\n") + for _, attr := range attributes { + for _, line := range mstOutputLines { + if strings.Contains(line, attr) { + regexResult := formatRegex.FindStringSubmatch(line) + fwCurrent[attr] = regexResult[3] + fwNext[attr] = regexResult[4] + break + } + } + } + return +} + +func HasMellanoxInterfacesInSpec(ifaceStatuses sriovnetworkv1.InterfaceExts, ifaceSpecs sriovnetworkv1.Interfaces) bool { + for _, ifaceStatus := range ifaceStatuses { + if ifaceStatus.Vendor == VendorMellanox { + for _, iface := range ifaceSpecs { + if iface.PciAddress == ifaceStatus.PciAddress { + log.Log.V(2).Info("hasMellanoxInterfacesInSpec(): Mellanox device specified in SriovNetworkNodeState spec", + "name", ifaceStatus.Name, + "address", ifaceStatus.PciAddress) + return true + } + } + } + } + return false +} + +func GetPciAddressPrefix(pciAddress string) string { + return pciAddress[:len(pciAddress)-1] +} + +func IsDualPort(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) bool { + log.Log.Info("mellanox-plugin IsDualPort()", "address", pciAddress) + pciAddressPrefix := GetPciAddressPrefix(pciAddress) + return len(mellanoxNicsStatus[pciAddressPrefix]) > 1 +} + +// handleTotalVfs return required total VFs or max (required VFs for dual port NIC) and needReboot if totalVfs will change +func HandleTotalVfs(fwCurrent, fwNext, attrs *MlxNic, ifaceSpec sriovnetworkv1.Interface, isDualPort bool, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) ( + totalVfs int, needReboot, changeWithoutReboot bool) { + totalVfs = ifaceSpec.NumVfs + // Check if the other port is changing theGetMlnxNicFwData number of VF + if isDualPort { + otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress, mellanoxNicsSpec) + if otherIfaceSpec != nil { + if otherIfaceSpec.NumVfs > totalVfs { + totalVfs = otherIfaceSpec.NumVfs + } + } + } + + // if the PF is externally managed we just need to check the totalVfs requested in the policy is not higher than + // the configured amount + if ifaceSpec.ExternallyManaged { + if totalVfs > fwCurrent.TotalVfs { + log.Log.Error(nil, "The nic is externallyManaged and TotalVfs configured on the system is lower then requested VFs, failing configuration", + "current", fwCurrent.TotalVfs, "requested", totalVfs) + attrs.TotalVfs = totalVfs + needReboot = true + changeWithoutReboot = false + } + return + } + + if fwCurrent.TotalVfs != totalVfs { + log.Log.V(2).Info("Changing TotalVfs, needs reboot", "current", fwCurrent.TotalVfs, "requested", totalVfs) + attrs.TotalVfs = totalVfs + needReboot = true + } + + // Remove policy then re-apply it + if !needReboot && fwNext.TotalVfs != totalVfs { + log.Log.V(2).Info("Changing TotalVfs to same as Next Boot value, doesn't require rebooting", + "current", fwCurrent.TotalVfs, "next", fwNext.TotalVfs, "requested", totalVfs) + attrs.TotalVfs = totalVfs + changeWithoutReboot = true + } + + return +} + +// handleEnableSriov based on totalVfs it decide to disable (totalVfs=0) or enable (totalVfs changed from 0) sriov +// and need reboot if enableSriov will change +func HandleEnableSriov(totalVfs int, fwCurrent, fwNext, attrs *MlxNic) (needReboot, changeWithoutReboot bool) { + if totalVfs == 0 && fwCurrent.EnableSriov { + log.Log.V(2).Info("disabling Sriov, needs reboot") + attrs.EnableSriov = false + return true, false + } else if totalVfs > 0 && !fwCurrent.EnableSriov { + log.Log.V(2).Info("enabling Sriov, needs reboot") + attrs.EnableSriov = true + return true, false + } else if totalVfs > 0 && !fwNext.EnableSriov { + attrs.EnableSriov = true + return false, true + } + + return false, false +} + +// handleLinkType based on existing linkType and requested link +func HandleLinkType(pciPrefix string, fwData, attr *MlxNic, + mellanoxNicsSpec map[string]sriovnetworkv1.Interface, + mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) (bool, error) { + needReboot := false + + pciAddress := pciPrefix + "0" + if firstPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok { + ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus) + needChange, err := isLinkTypeRequireChange(firstPortSpec, ifaceStatus, fwData.LinkTypeP1) + if err != nil { + return false, err + } + + if needChange { + log.Log.V(2).Info("Changing LinkTypeP1, needs reboot", + "from", fwData.LinkTypeP1, "to", firstPortSpec.LinkType) + attr.LinkTypeP1 = firstPortSpec.LinkType + needReboot = true + } + } + + pciAddress = pciPrefix + "1" + if secondPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok { + ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus) + needChange, err := isLinkTypeRequireChange(secondPortSpec, ifaceStatus, fwData.LinkTypeP2) + if err != nil { + return false, err + } + + if needChange { + log.Log.V(2).Info("Changing LinkTypeP2, needs reboot", + "from", fwData.LinkTypeP2, "to", secondPortSpec.LinkType) + attr.LinkTypeP2 = secondPortSpec.LinkType + needReboot = true + } + } + + return needReboot, nil +} + +func mlnxNicFromMap(mstData map[string]string) (*MlxNic, error) { + log.Log.Info("mellanox-plugin mlnxNicFromMap()", "data", mstData) + fwData := &MlxNic{} + if strings.Contains(mstData[EnableSriov], "True") { + fwData.EnableSriov = true + } + i, err := strconv.Atoi(mstData[TotalVfs]) + if err != nil { + return nil, err + } + + fwData.TotalVfs = i + fwData.LinkTypeP1 = getLinkType(mstData[LinkTypeP1]) + if linkTypeP2, ok := mstData[LinkTypeP2]; ok { + fwData.LinkTypeP2 = getLinkType(linkTypeP2) + } + + return fwData, nil +} + +func getLinkType(linkType string) string { + log.Log.Info("mellanox-plugin getLinkType()", "link-type", linkType) + if strings.Contains(linkType, consts.LinkTypeETH) { + return consts.LinkTypeETH + } else if strings.Contains(linkType, consts.LinkTypeIB) { + return consts.LinkTypeIB + } else if len(linkType) > 0 { + log.Log.Error(nil, "mellanox-plugin getLinkType(): link type is not one of [ETH, IB], treating as unknown", + "link-type", linkType) + return UknownLinkType + } else { + log.Log.Info("mellanox-plugin getLinkType(): LINK_TYPE_P* attribute was not found, treating as preconfigured link type") + return PreconfiguredLinkType + } +} + +func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt, fwLinkType string) (bool, error) { + log.Log.Info("mellanox-plugin isLinkTypeRequireChange()", "device", iface.PciAddress) + if iface.LinkType != "" && !strings.EqualFold(ifaceStatus.LinkType, iface.LinkType) { + if !strings.EqualFold(iface.LinkType, consts.LinkTypeETH) && !strings.EqualFold(iface.LinkType, consts.LinkTypeIB) { + return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Not supported link type: %s,"+ + " supported link types: [eth, ETH, ib, and IB]", iface.LinkType) + } + if fwLinkType == UknownLinkType { + return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Unknown link type: %s", fwLinkType) + } + if fwLinkType == PreconfiguredLinkType { + return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Network card %s does not support link type change", iface.PciAddress) + } + + return true, nil + } + + return false, nil +} + +func getOtherPortSpec(pciAddress string, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) *sriovnetworkv1.Interface { + log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress) + pciAddrPrefix := GetPciAddressPrefix(pciAddress) + pciAddrSuffix := pciAddress[len(pciAddrPrefix):] + + if pciAddrSuffix == "0" { + iface := mellanoxNicsSpec[pciAddrPrefix+"1"] + return &iface + } + + iface := mellanoxNicsSpec[pciAddrPrefix+"0"] + return &iface +} + +func getIfaceStatus(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) sriovnetworkv1.InterfaceExt { + return mellanoxNicsStatus[GetPciAddressPrefix(pciAddress)][pciAddress] +} diff --git a/pkg/vendors/mellanox/mock/mock_mellanox.go b/pkg/vendors/mellanox/mock/mock_mellanox.go new file mode 100644 index 000000000..78c1d4903 --- /dev/null +++ b/pkg/vendors/mellanox/mock/mock_mellanox.go @@ -0,0 +1,96 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: mellanox.go + +// Package mock_mlxutils is a generated GoMock package. +package mock_mlxutils + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + mlxutils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" +) + +// MockMellanoxInterface is a mock of MellanoxInterface interface. +type MockMellanoxInterface struct { + ctrl *gomock.Controller + recorder *MockMellanoxInterfaceMockRecorder +} + +// MockMellanoxInterfaceMockRecorder is the mock recorder for MockMellanoxInterface. +type MockMellanoxInterfaceMockRecorder struct { + mock *MockMellanoxInterface +} + +// NewMockMellanoxInterface creates a new mock instance. +func NewMockMellanoxInterface(ctrl *gomock.Controller) *MockMellanoxInterface { + mock := &MockMellanoxInterface{ctrl: ctrl} + mock.recorder = &MockMellanoxInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMellanoxInterface) EXPECT() *MockMellanoxInterfaceMockRecorder { + return m.recorder +} + +// GetMellanoxBlueFieldMode mocks base method. +func (m *MockMellanoxInterface) GetMellanoxBlueFieldMode(arg0 string) (mlxutils.BlueFieldMode, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMellanoxBlueFieldMode", arg0) + ret0, _ := ret[0].(mlxutils.BlueFieldMode) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMellanoxBlueFieldMode indicates an expected call of GetMellanoxBlueFieldMode. +func (mr *MockMellanoxInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMellanoxBlueFieldMode", reflect.TypeOf((*MockMellanoxInterface)(nil).GetMellanoxBlueFieldMode), arg0) +} + +// GetMlxNicFwData mocks base method. +func (m *MockMellanoxInterface) GetMlxNicFwData(pciAddress string) (*mlxutils.MlxNic, *mlxutils.MlxNic, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMlxNicFwData", pciAddress) + ret0, _ := ret[0].(*mlxutils.MlxNic) + ret1, _ := ret[1].(*mlxutils.MlxNic) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetMlxNicFwData indicates an expected call of GetMlxNicFwData. +func (mr *MockMellanoxInterfaceMockRecorder) GetMlxNicFwData(pciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMlxNicFwData", reflect.TypeOf((*MockMellanoxInterface)(nil).GetMlxNicFwData), pciAddress) +} + +// MlxConfigFW mocks base method. +func (m *MockMellanoxInterface) MlxConfigFW(attributesToChange map[string]mlxutils.MlxNic) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MlxConfigFW", attributesToChange) + ret0, _ := ret[0].(error) + return ret0 +} + +// MlxConfigFW indicates an expected call of MlxConfigFW. +func (mr *MockMellanoxInterfaceMockRecorder) MlxConfigFW(attributesToChange interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxConfigFW", reflect.TypeOf((*MockMellanoxInterface)(nil).MlxConfigFW), attributesToChange) +} + +// MstConfigReadData mocks base method. +func (m *MockMellanoxInterface) MstConfigReadData(arg0 string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MstConfigReadData", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// MstConfigReadData indicates an expected call of MstConfigReadData. +func (mr *MockMellanoxInterfaceMockRecorder) MstConfigReadData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MstConfigReadData", reflect.TypeOf((*MockMellanoxInterface)(nil).MstConfigReadData), arg0) +} diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index 3e5606156..f225dd6c2 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -16,8 +16,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) const ( @@ -35,7 +35,7 @@ func validateSriovOperatorConfig(cr *sriovnetworkv1.SriovOperatorConfig, operati log.Log.V(2).Info("validateSriovOperatorConfig", "object", cr) var warnings []string - if cr.GetName() != constants.DefaultConfigName { + if cr.GetName() != consts.DefaultConfigName { return false, warnings, fmt.Errorf("only default SriovOperatorConfig is used") } @@ -95,7 +95,7 @@ func validateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePolicy, o log.Log.V(2).Info("validateSriovNetworkNodePolicy", "object", cr) var warnings []string - if cr.GetName() == constants.DefaultPolicyName && cr.GetNamespace() == os.Getenv("NAMESPACE") { + if cr.GetName() == consts.DefaultPolicyName && cr.GetNamespace() == os.Getenv("NAMESPACE") { if operation == v1.Delete { warnings = append(warnings, "default SriovNetworkNodePolicy shouldn't be deleted") } @@ -191,19 +191,19 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol // To configure RoCE on baremetal or virtual machine: // BM: DeviceType = netdevice && isRdma = true // VM: DeviceType = vfio-pci && isRdma = false - if cr.Spec.DeviceType == constants.DeviceTypeVfioPci && cr.Spec.IsRdma { + if cr.Spec.DeviceType == consts.DeviceTypeVfioPci && cr.Spec.IsRdma { return false, fmt.Errorf("'deviceType: vfio-pci' conflicts with 'isRdma: true'; Set 'deviceType' to (string)'netdevice' Or Set 'isRdma' to (bool)'false'") } - if strings.EqualFold(cr.Spec.LinkType, constants.LinkTypeIB) && !cr.Spec.IsRdma { + if strings.EqualFold(cr.Spec.LinkType, consts.LinkTypeIB) && !cr.Spec.IsRdma { return false, fmt.Errorf("'linkType: ib or IB' requires 'isRdma: true'; Set 'isRdma' to (bool)'true'") } // vdpa: deviceType must be set to 'netdevice' - if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) { + if cr.Spec.DeviceType != consts.DeviceTypeNetDevice && (cr.Spec.VdpaType == consts.VdpaTypeVirtio || cr.Spec.VdpaType == consts.VdpaTypeVhost) { return false, fmt.Errorf("'deviceType: %s' conflicts with '%s'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType, cr.Spec.VdpaType) } // vdpa: device must be configured in switchdev mode - if (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { + if (cr.Spec.VdpaType == consts.VdpaTypeVirtio || cr.Spec.VdpaType == consts.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode") } @@ -295,7 +295,7 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s if err == nil { interfaceSelected = true interfaceSelectedForNode = true - if policy.GetName() != constants.DefaultPolicyName && policy.Spec.NumVfs == 0 { + if policy.GetName() != consts.DefaultPolicyName && policy.Spec.NumVfs == 0 { return nil, fmt.Errorf("numVfs(%d) in CR %s is not allowed", policy.Spec.NumVfs, policy.GetName()) } if policy.Spec.NumVfs > iface.TotalVfs && iface.Vendor == IntelID { @@ -320,7 +320,7 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s } } // vdpa: only mellanox cards are supported - if (policy.Spec.VdpaType == constants.VdpaTypeVirtio || policy.Spec.VdpaType == constants.VdpaTypeVhost) && iface.Vendor != MellanoxID { + if (policy.Spec.VdpaType == consts.VdpaTypeVirtio || policy.Spec.VdpaType == consts.VdpaTypeVhost) && iface.Vendor != MellanoxID { return nil, fmt.Errorf("vendor(%s) in CR %s not supported for vdpa interface(%s)", iface.Vendor, policy.GetName(), iface.Name) } } else { @@ -438,7 +438,7 @@ func validateNicModel(selector *sriovnetworkv1.SriovNetworkNicSelector, iface *s } // Check the vendor and device ID of the VF only if we are on a virtual environment - for key := range utils.PlatformMap { + for key := range vars.PlatformsMap { if strings.Contains(strings.ToLower(node.Spec.ProviderID), strings.ToLower(key)) && selector.NetFilter != "" && selector.NetFilter == iface.NetFilter && sriovnetworkv1.IsVfSupportedModel(iface.Vendor, iface.DeviceID) { diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index 809a2c1bd..208ed2b9f 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -1193,7 +1193,7 @@ var _ = Describe("[sriov] operator", func() { capacity, _ = resNum.AsInt64() res["openshift.io/testresource1"] = capacity return res - }, 2*time.Minute, time.Second).Should(Equal(map[string]int64{ + }, 15*time.Minute, time.Second).Should(Equal(map[string]int64{ "openshift.io/testresource": int64(3), "openshift.io/testresource1": int64(2), })) diff --git a/test/util/cluster/cluster.go b/test/util/cluster/cluster.go index fdf06bdf7..af2230cff 100644 --- a/test/util/cluster/cluster.go +++ b/test/util/cluster/cluster.go @@ -17,6 +17,7 @@ import ( runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/nodes" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod" @@ -323,7 +324,7 @@ func GetNodeSecureBootState(clients *testclient.ClientSet, nodeName, namespace s podDefinition.Namespace = namespace volume := corev1.Volume{Name: "host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/"}}} - mount := corev1.VolumeMount{Name: "host", MountPath: "/host"} + mount := corev1.VolumeMount{Name: "host", MountPath: consts.Host} podDefinition = pod.RedefineWithMount(podDefinition, volume, mount) created, err := clients.Pods(namespace).Create(context.Background(), podDefinition, metav1.CreateOptions{}) if err != nil {