diff --git a/Makefile b/Makefile index 99fc04a0b9..94f4464f8a 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/api/v1/helper.go b/api/v1/helper.go index 2042465afe..4559f0451c 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -24,10 +24,11 @@ import ( ) const ( - LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" - FINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" - ESWITCHMODE_LEGACY = "legacy" - ESWITCHMODE_SWITCHDEV = "switchdev" + LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" + NETATTDEFFINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" + POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io" + ESWITCHMODE_LEGACY = "legacy" + ESWITCHMODE_SWITCHDEV = "switchdev" ) const invalidVfIndex = -1 diff --git a/controllers/helper.go b/controllers/helper.go index 0a30683517..947125c1ed 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -22,21 +22,21 @@ import ( ) const ( - ResyncPeriod = 5 * time.Minute - DEFAULT_CONFIG_NAME = "default" - CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" - INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" - OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" - SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" - INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" - OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" - DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" - PLUGIN_PATH = "./bindata/manifests/plugins" - DAEMON_PATH = "./bindata/manifests/daemon" - DEFAULT_POLICY_NAME = "default" - CONFIGMAP_NAME = "device-plugin-config" - DP_CONFIG_FILENAME = "config.json" - HwOffloadNodeLabel = "ovs-hw-offload-worker" + ResyncPeriod = 5 * time.Minute + DEFAULT_CONFIG_NAME = "default" + CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" + INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" + OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" + SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" + INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" + OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" + DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" + PLUGIN_PATH = "./bindata/manifests/plugins" + DAEMON_PATH = "./bindata/manifests/daemon" + DEFAULT_POLICY_NAME = "default" + CONFIGMAP_NAME = "device-plugin-config" + DP_CONFIG_FILENAME = "config.json" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX = "ovs-hw-offload" linkTypeEthernet = "ether" linkTypeInfiniband = "infiniband" diff --git a/controllers/sriovibnetwork_controller.go b/controllers/sriovibnetwork_controller.go index 63f73af4fd..28cf9bd32c 100644 --- a/controllers/sriovibnetwork_controller.go +++ b/controllers/sriovibnetwork_controller.go @@ -74,15 +74,15 @@ func (r *SriovIBNetworkReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { - instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.FINALIZERNAME) + if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } } } else { // The object is being deleted - if sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { + if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { // our finalizer is present, so lets handle any external dependency reqLogger.Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.Spec.NetworkNamespace, "Name", instance.Name) if err := instance.DeleteNetAttDef(r); err != nil { @@ -91,7 +91,7 @@ func (r *SriovIBNetworkReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err return reconcile.Result{}, err } // remove our finalizer from the list and update it. - instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } diff --git a/controllers/sriovnetwork_controller.go b/controllers/sriovnetwork_controller.go index 588aae77a7..fe94634bc3 100644 --- a/controllers/sriovnetwork_controller.go +++ b/controllers/sriovnetwork_controller.go @@ -74,15 +74,15 @@ func (r *SriovNetworkReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { - instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.FINALIZERNAME) + if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } } } else { // The object is being deleted - if sriovnetworkv1.StringInArray(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) { + if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) { // our finalizer is present, so lets handle any external dependency reqLogger.Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.Spec.NetworkNamespace, "Name", instance.Name) if err := instance.DeleteNetAttDef(r); err != nil { @@ -91,7 +91,7 @@ func (r *SriovNetworkReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error return reconcile.Result{}, err } // remove our finalizer from the list and update it. - instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.FINALIZERNAME, instance.ObjectMeta.Finalizers) + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) if err := r.Update(context.Background(), instance); err != nil { return reconcile.Result{}, err } diff --git a/controllers/sriovnetworknodeconfig_controller.go b/controllers/sriovnetworknodeconfig_controller.go deleted file mode 100644 index b51665e3fe..0000000000 --- a/controllers/sriovnetworknodeconfig_controller.go +++ /dev/null @@ -1,47 +0,0 @@ -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovNetworkNodeConfigReconciler reconciles a SriovNetworkNodeConfig object -type SriovNetworkNodeConfigReconciler struct { - client.Client - Log logr.Logger - Scheme *runtime.Scheme -} - -//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworknodeconfigs/status,verbs=get;update;patch - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the SriovNetworkNodeConfig object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile -func (r *SriovNetworkNodeConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - _ = context.Background() - _ = r.Log.WithValues("sriovnetworknodeconfig", req.NamespacedName) - - // your logic here - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *SriovNetworkNodeConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&sriovnetworkv1.SriovNetworkNodeConfig{}). - Complete(r) -} diff --git a/controllers/sriovnetworkpoolconfig_controller.go b/controllers/sriovnetworkpoolconfig_controller.go new file mode 100644 index 0000000000..bac6179a95 --- /dev/null +++ b/controllers/sriovnetworkpoolconfig_controller.go @@ -0,0 +1,205 @@ +package controllers + +import ( + "bytes" + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +// SriovNetworkPoolConfigReconciler reconciles a SriovNetworkPoolConfig object +type SriovNetworkPoolConfigReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovnetworkpoolconfigs/status,verbs=get;update;patch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the SriovNetworkPoolConfig object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile +func (r *SriovNetworkPoolConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + _ = context.Background() + reqLogger := r.Log.WithValues("sriovnetworkpoolconfig", req.NamespacedName) + reqLogger.Info("Reconciling") + + // // Fetch SriovNetworkPoolConfig + instance := &sriovnetworkv1.SriovNetworkPoolConfig{} + err := r.Get(context.TODO(), req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + reqLogger.Info("instance not found") + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // examine DeletionTimestamp to determine if object is under deletion + if instance.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !sriovnetworkv1.StringInArray(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) { + instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, sriovnetworkv1.POOLCONFIGFINALIZERNAME) + if err := r.Update(context.Background(), instance); err != nil { + return reconcile.Result{}, err + } + } + if utils.ClusterType == utils.ClusterTypeOpenshift { + if err = r.syncOvsHardwareOffloadMachineConfigs(instance, false); err != nil { + return reconcile.Result{}, err + } + } + } else { + // The object is being deleted + if sriovnetworkv1.StringInArray(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) { + // our finalizer is present, so lets handle any external dependency + reqLogger.Info("delete SriovNetworkPoolConfig CR", "Namespace", instance.Namespace, "Name", instance.Name) + if utils.ClusterType == utils.ClusterTypeOpenshift { + if err = r.syncOvsHardwareOffloadMachineConfigs(instance, true); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return reconcile.Result{}, err + } + } + // remove our finalizer from the list and update it. + instance.ObjectMeta.Finalizers = sriovnetworkv1.RemoveString(sriovnetworkv1.POOLCONFIGFINALIZERNAME, instance.ObjectMeta.Finalizers) + if err := r.Update(context.Background(), instance); err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, err + } + + return reconcile.Result{RequeueAfter: ResyncPeriod}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SriovNetworkPoolConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sriovnetworkv1.SriovNetworkPoolConfig{}). + Complete(r) +} + +func (r *SriovNetworkPoolConfigReconciler) syncOvsHardwareOffloadMachineConfigs(nc *sriovnetworkv1.SriovNetworkPoolConfig, deletion bool) error { + logger := r.Log.WithName("syncOvsHardwareOffloadMachineConfigs") + + data := render.MakeRenderData() + mcpMap := make(map[string]bool) + + mcpList := &mcfgv1.MachineConfigPoolList{} + err := r.List(context.TODO(), mcpList, &client.ListOptions{}) + if err != nil { + return fmt.Errorf("Failed to get MachineConfigPoolList: %v", err) + } + + for _, mcp := range mcpList.Items { + // MachineConfigPool is selected when MachineConfigPool name matches with OvsHardwareOffloadConfig.Name + if mcp.GetName() == nc.Spec.OvsHardwareOffloadConfig.Name { + if mcp.GetName() == "master" { + logger.Info("Master nodes are selected viby OvsHardwareOffloadConfig.Name, ignoring.") + continue + } + if deletion { + mcpMap[mcp.GetName()] = false + } else { + mcpMap[mcp.GetName()] = true + } + break + } + } + + for mcpName, enable := range mcpMap { + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, mcpName, true, &data) + if err != nil { + return err + } + + foundMC := &mcfgv1.MachineConfig{} + err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) + if err != nil { + if errors.IsNotFound(err) { + if enable { + err = r.Create(context.TODO(), mc) + if err != nil { + return fmt.Errorf("Couldn't create MachineConfig: %v", err) + } + logger.Info("Created MachineConfig CR in MachineConfigPool", mcName, mcpName) + } + } else { + return fmt.Errorf("Failed to get MachineConfig: %v", err) + } + } else { + if enable { + if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { + logger.Info("MachineConfig already exists, updating") + err = r.Update(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't update MachineConfig: %v", err) + } + } else { + logger.Info("No content change, skip updating MC") + } + } else { + logger.Info("offload disabled, delete MachineConfig") + err = r.Delete(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't delete MachineConfig: %v", err) + } + } + } + } + + // Remove stale MCs for MCP that no longer exists in OvsHardwareOffloadConfig + for _, mcp := range mcpList.Items { + found := false + for mcpName := range mcpMap { + if mcp.Name == mcpName { + found = true + break + } + } + if !found { + mcName := "00-" + mcp.Name + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + foundMC := &mcfgv1.MachineConfig{} + err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) + if err == nil { + delErr := r.Delete(context.TODO(), foundMC) + if delErr != nil { + return fmt.Errorf("Couldn't delete MachineConfig: %v", delErr) + } + } else { + if !errors.IsNotFound(err) { + return fmt.Errorf("Failed to get MachineConfig: %v", err) + } + } + } + } + + return nil +} diff --git a/controllers/sriovnetworkpoolconfig_controller_test.go b/controllers/sriovnetworkpoolconfig_controller_test.go new file mode 100644 index 0000000000..e7d38e84a6 --- /dev/null +++ b/controllers/sriovnetworkpoolconfig_controller_test.go @@ -0,0 +1,63 @@ +package controllers + +import ( + goctx "context" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +var _ = Describe("Operator", func() { + Context("When is up", func() { + It("should be able to create machine config for MachineConfigPool specified in sriov pool config", func() { + config := &sriovnetworkv1.SriovNetworkPoolConfig{} + config.SetNamespace(testNamespace) + config.SetName("ovs-hw-offload-config") + mcpName := "worker" + mc := &mcfgv1.MachineConfig{} + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp := &mcfgv1.MachineConfigPool{} + err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp = &mcfgv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcpName, + }, + Spec: mcfgv1.MachineConfigPoolSpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/worker": "", + }, + }, + }, + } + err = k8sClient.Create(goctx.TODO(), mcp) + Expect(err).NotTo(HaveOccurred()) + + config.Spec.OvsHardwareOffloadConfig = sriovnetworkv1.OvsHardwareOffloadConfig{ + Name: mcpName, + } + err = k8sClient.Create(goctx.TODO(), config) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() error { + mc := &mcfgv1.MachineConfig{} + err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + if err != nil { + return err + } + return nil + }, timeout*3, interval).Should(Succeed()) + }) + }) +}) diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 5e35919126..5fddc3782c 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -17,14 +17,11 @@ limitations under the License. package controllers import ( - "bytes" "context" "fmt" "os" "github.com/go-logr/logr" - "github.com/openshift/machine-config-operator/lib/resourcemerge" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -111,12 +108,6 @@ func (r *SriovOperatorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result return reconcile.Result{}, err } - if utils.ClusterType == utils.ClusterTypeOpenshift { - if err = r.syncOffloadMachineConfig(defaultConfig); err != nil { - return reconcile.Result{}, err - } - } - return reconcile.Result{RequeueAfter: ResyncPeriod}, nil } @@ -288,102 +279,3 @@ func (r *SriovOperatorConfigReconciler) syncK8sResource(cr *sriovnetworkv1.Sriov } return nil } - -func (r *SriovOperatorConfigReconciler) syncOffloadMachineConfig(dc *sriovnetworkv1.SriovOperatorConfig) error { - logger := r.Log.WithName("syncOffloadMachineConfig") - var err error - - logger.Info("Start to render MachineConfig and MachineConfigPool for OVS HW offloading") - data := render.MakeRenderData() - data.Data["HwOffloadNodeLabel"] = HwOffloadNodeLabel - mcName := "00-" + HwOffloadNodeLabel - mcpName := HwOffloadNodeLabel - mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, HwOffloadNodeLabel, dc.Spec.EnableOvsOffload, &data) - if err != nil { - return err - } - mcpRaw, err := render.RenderTemplate("bindata/manifests/switchdev-config/machineconfigpool.yaml", &data) - if err != nil { - return err - } - mcp := &mcfgv1.MachineConfigPool{} - if len(mcpRaw) != 1 { - return fmt.Errorf("Invalid MachineConfigPool CR template") - } - err = r.Scheme.Convert(mcpRaw[0], mcp, context.TODO()) - if err != nil { - return err - } - - foundMC := &mcfgv1.MachineConfig{} - foundMCP := &mcfgv1.MachineConfigPool{} - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mc) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfig: %v", err) - } - logger.Info("Created MachineConfig CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfig: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MC") - } - } else { - logger.Info("offload disabled, delete MachineConfig") - err = r.Delete(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfig: %v", err) - } - } - } - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcpName}, foundMCP) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mcp) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfigPool: %v", err) - } - logger.Info("Created MachineConfigPool CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfigPool: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - modified := resourcemerge.BoolPtr(false) - resourcemerge.EnsureMachineConfigPool(modified, foundMCP, *mcp) - if *modified { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MCP") - } - } else { - logger.Info("offload disabled, delete MachineConfigPool") - err = r.Delete(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfigPool: %v", err) - } - } - } - return nil -} diff --git a/controllers/sriovoperatorconfig_controller_test.go b/controllers/sriovoperatorconfig_controller_test.go index b1b0a33a8c..bd2919e153 100644 --- a/controllers/sriovoperatorconfig_controller_test.go +++ b/controllers/sriovoperatorconfig_controller_test.go @@ -5,13 +5,11 @@ import ( admv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" util "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" @@ -174,64 +172,5 @@ var _ = Describe("Operator", func() { }, timeout*10, interval).Should(Equal(config.Spec.ConfigDaemonNodeSelector)) }) - It("should be able to create/remove hw offload MCO and MC", func() { - config := &sriovnetworkv1.SriovOperatorConfig{} - err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", interval, timeout) - Expect(err).NotTo(HaveOccurred()) - By("by default") - mcName := "00-" + HwOffloadNodeLabel - mc := &mcfgv1.MachineConfig{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - mcpName := HwOffloadNodeLabel - mcp := &mcfgv1.MachineConfigPool{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - By("set EnableOvsOffload to true to create MC and MCP resources") - config.Spec.EnableOvsOffload = true - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - Eventually(func() error { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - - By("set EnableOvsOffload to false to delete MC and MCP resources") - config.Spec.EnableOvsOffload = false - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() bool { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - Eventually(func() bool { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - }) }) }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 8e34108dde..3e24a3bc70 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -118,6 +118,13 @@ var _ = BeforeSuite(func(done Done) { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&SriovNetworkPoolConfigReconciler{ + Client: k8sManager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("SriovNetworkPoolConfig"), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + os.Setenv("RESOURCE_PREFIX", "openshift.io") os.Setenv("NAMESPACE", "openshift-sriov-network-operator") os.Setenv("ENABLE_ADMISSION_CONTROLLER", "true") @@ -156,6 +163,12 @@ var _ = BeforeSuite(func(done Done) { LogLevel: 2, } Expect(k8sClient.Create(context.TODO(), config)).Should(Succeed()) + + poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{} + poolConfig.SetNamespace(testNamespace) + poolConfig.SetName(DEFAULT_CONFIG_NAME) + poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{} + Expect(k8sClient.Create(context.TODO(), poolConfig)).Should(Succeed()) close(done) }, 60) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index f4a07095a2..672e5c95b7 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -464,7 +464,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { reqReboot := false reqDrain := false - mcoReboot := false for k, p := range dn.LoadedPlugins { d, r := false, false if dn.nodeState.GetName() == "" { @@ -476,11 +475,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.Errorf("nodeStateSyncHandler(): plugin %s error: %v", k, err) return err } - - if p.Name() == "mco_plugin" && d && !r { - mcoReboot = true - } - glog.V(0).Infof("nodeStateSyncHandler(): plugin %s: reqDrain %v, reqReboot %v", k, d, r) reqDrain = reqDrain || d reqReboot = reqReboot || r @@ -488,7 +482,7 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.V(0).Infof("nodeStateSyncHandler(): reqDrain %v, reqReboot %v disableDrain %v", reqDrain, reqReboot, dn.disableDrain) for k, p := range dn.LoadedPlugins { - if k != GenericPlugin && k != McoPlugin { + if k != GenericPlugin { err := p.Apply() if err != nil { glog.Errorf("nodeStateSyncHandler(): plugin %s fail to apply: %v", k, err) @@ -496,15 +490,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } } } - if mcoReboot { - glog.V(0).Infof("nodeStateSyncHandler(): MCO will trigger reboot") - err = dn.LoadedPlugins[McoPlugin].Apply() - if err != nil { - glog.Errorf("nodeStateSyncHandler(): mco_plugin fail to apply: %v", err) - return err - } - return nil - } if utils.ClusterType == utils.ClusterTypeOpenshift { if err = dn.getNodeMachinePool(); err != nil { return err @@ -649,9 +634,7 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = append(pl, VirtualPlugin) } else { pl = registerPlugins(ns) - if utils.ClusterType == utils.ClusterTypeOpenshift { - pl = append(pl, McoPlugin) - } else { + if utils.ClusterType != utils.ClusterTypeOpenshift { pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index ffab8607d9..eb4458f1bd 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -30,7 +30,6 @@ const ( SpecVersion = "1.0" GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" - McoPlugin = "mco_plugin" K8sPlugin = "k8s_plugin" ) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 966fdc8e0c..97d3d07ac0 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -59,18 +59,10 @@ func (p *GenericPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeSta p.DesireState = state needDrain = needDrainNode(state.Spec.Interfaces, state.Status.Interfaces) + needReboot = needRebootNode(state, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(state) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -84,18 +76,10 @@ func (p *GenericPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkN p.DesireState = new needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces) + needReboot = needRebootNode(new, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(new) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -212,3 +196,32 @@ func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.Int } return } + +func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint) (needReboot bool) { + needReboot = false + if *loadVfioDriver != loaded { + if needVfioDriver(state) { + *loadVfioDriver = loading + update, err := tryEnableIommuInKernelArgs() + if err != nil { + glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err) + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args") + } + needReboot = needReboot || update + } + } + + if utils.IsSwitchdevModeSpec(state.Spec) { + update, err := utils.WriteSwitchdevConfFile(state) + if err != nil { + glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file") + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration") + } + needReboot = needReboot || update + } + return +} diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index c26d82b42f..e3def1e000 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -117,34 +117,18 @@ func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS p.updateTarget.reset() // TODO add check for enableOvsOffload in OperatorConfig later - // Update services if switchdev not required - if utils.IsSwitchdevModeSpec(new.Spec) { - // Check services - err = p.servicesStateUpdate() - if err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) - return - } - } - - // Check switchdev config - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + // Update services if switchdev required + if !utils.IsSwitchdevModeSpec(new.Spec) { return } - if remove { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") - needDrain = true - needReboot = true - return - } - if update { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true + + // Check services + err = p.servicesStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) return } + if p.updateTarget.needUpdate() { needDrain = true if p.updateTarget.switchdevUdevScript { diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go deleted file mode 100644 index b63c8dfe31..0000000000 --- a/pkg/plugins/mco/mco_plugin.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - - "github.com/golang/glog" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -type McoPlugin struct { - PluginName string - SpecVersion string - DesireState *sriovnetworkv1.SriovNetworkNodeState - LastState *sriovnetworkv1.SriovNetworkNodeState -} - -const ( - switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - nodeLabelPrefix = "node-role.kubernetes.io/" -) - -var nodeName string -var Plugin McoPlugin -var kubeclient *kubernetes.Clientset -var switchdevConfigured bool - -// Initialize our plugin and set up initial values -func init() { - Plugin = McoPlugin{ - PluginName: "mco_plugin", - SpecVersion: "1.0", - } - - var config *rest.Config - var err error - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig != "" { - config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - } else { - // creates the in-cluster config - config, err = rest.InClusterConfig() - } - - if err != nil { - panic(err.Error()) - } - kubeclient = kubernetes.NewForConfigOrDie(config) -} - -// Name returns the name of the plugin -func (p *McoPlugin) Name() string { - return p.PluginName -} - -// Spec returns the version of the spec expected by the plugin -func (p *McoPlugin) Spec() string { - return p.SpecVersion -} - -// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateAdd()") - nodeName = state.GetName() - return p.OnNodeStateChange(nil, state) -} - -// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateChange()") - switchdevConfigured = utils.IsSwitchdevModeSpec(new.Spec) - - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) - return - } - if remove { - needDrain = true - return - } - if update { - if _, err = os.Stat(switchdevUnitPath); err != nil { - if os.IsNotExist(err) { - glog.Info("mco-plugin OnNodeStateChange(): the latest MachineConfig has not been applied") - needDrain = true - err = nil - return - } - return - } - // node is already in the offload MCP - glog.Info("mco-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true - return - } - return -} - -// Apply config change -func (p *McoPlugin) Apply() error { - glog.Info("mco-plugin Apply()") - node, err := kubeclient.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - label := nodeLabelPrefix + controllers.HwOffloadNodeLabel - if switchdevConfigured { - if _, ok := node.Labels[label]; !ok { - glog.Infof("Move node %s into HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: "", - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is already in HW offload MachineConfigPool", node.Name) - return nil - } - - if _, ok := node.Labels[label]; ok { - glog.Infof("Remove node %s from HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: nil, - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) - return nil -} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go index 26d7819a07..c94373178b 100644 --- a/pkg/utils/switchdev.go +++ b/pkg/utils/switchdev.go @@ -23,7 +23,7 @@ func IsSwitchdevModeSpec(spec sriovnetworkv1.SriovNetworkNodeStateSpec) bool { return false } -func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update bool, err error) { _, err = os.Stat(switchDevConfPath) if err != nil { if os.IsNotExist(err) { @@ -52,10 +52,6 @@ func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (upd glog.V(2).Info("WriteSwitchdevConfFile(): no update") return } - if newContent == "" { - remove = true - glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") - } update = true glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0644)