From f5c463c16b94a31a2ff1d6b7155046c2c0723830 Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Thu, 6 May 2021 16:39:43 +0800 Subject: [PATCH] OpenShift: enable OVS HW offload for multiple MCPs One node can only be added in one custom MCP pool by design of MCO. Since current ovs-hw-offload MCs are applied to hardcoded MCP ovs-hw-offload-worker, other existing custome MCPs cannot be enabled with ovs hw offload configuration. This commits allows ovs-hw-offload MCs be applied to multiple MCPs as specified in default SriovNetworkNodeConfig. --- Makefile | 2 +- controllers/helper.go | 30 ++-- .../sriovnetworknodeconfig_controller.go | 157 +++++++++++++++++- .../sriovnetworknodeconfig_controller_test.go | 67 ++++++++ controllers/sriovoperatorconfig_controller.go | 108 ------------ .../sriovoperatorconfig_controller_test.go | 61 ------- controllers/suite_test.go | 13 ++ main.go | 35 ++++ pkg/daemon/daemon.go | 21 +-- pkg/daemon/plugin.go | 1 - pkg/plugins/generic/generic_plugin.go | 56 ++++--- pkg/plugins/k8s/k8s_plugin.go | 32 +--- pkg/plugins/mco/mco_plugin.go | 150 ----------------- pkg/utils/switchdev.go | 6 +- 14 files changed, 330 insertions(+), 409 deletions(-) create mode 100644 controllers/sriovnetworknodeconfig_controller_test.go delete mode 100644 pkg/plugins/mco/mco_plugin.go diff --git a/Makefile b/Makefile index 08f078d518..4de52d8e27 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ _build-%: _plugin-%: vet @hack/build-plugins.sh $* -plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-mco _plugin-k8s +plugins: _plugin-intel _plugin-mellanox _plugin-generic _plugin-virtual _plugin-k8s clean: @rm -rf $(TARGET_DIR) diff --git a/controllers/helper.go b/controllers/helper.go index 0a30683517..947125c1ed 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -22,21 +22,21 @@ import ( ) const ( - ResyncPeriod = 5 * time.Minute - DEFAULT_CONFIG_NAME = "default" - CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" - INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" - OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" - SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" - INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" - OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" - DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" - PLUGIN_PATH = "./bindata/manifests/plugins" - DAEMON_PATH = "./bindata/manifests/daemon" - DEFAULT_POLICY_NAME = "default" - CONFIGMAP_NAME = "device-plugin-config" - DP_CONFIG_FILENAME = "config.json" - HwOffloadNodeLabel = "ovs-hw-offload-worker" + ResyncPeriod = 5 * time.Minute + DEFAULT_CONFIG_NAME = "default" + CONFIG_DAEMON_PATH = "./bindata/manifests/daemon" + INJECTOR_WEBHOOK_PATH = "./bindata/manifests/webhook" + OPERATOR_WEBHOOK_PATH = "./bindata/manifests/operator-webhook" + SERVICE_CA_CONFIGMAP_ANNOTATION = "service.beta.openshift.io/inject-cabundle" + INJECTOR_WEBHOOK_NAME = "network-resources-injector-config" + OPERATOR_WEBHOOK_NAME = "sriov-operator-webhook-config" + DEPRECATED_OPERATOR_WEBHOOK_NAME = "operator-webhook-config" + PLUGIN_PATH = "./bindata/manifests/plugins" + DAEMON_PATH = "./bindata/manifests/daemon" + DEFAULT_POLICY_NAME = "default" + CONFIGMAP_NAME = "device-plugin-config" + DP_CONFIG_FILENAME = "config.json" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX = "ovs-hw-offload" linkTypeEthernet = "ether" linkTypeInfiniband = "infiniband" diff --git a/controllers/sriovnetworknodeconfig_controller.go b/controllers/sriovnetworknodeconfig_controller.go index b51665e3fe..3018a0aaac 100644 --- a/controllers/sriovnetworknodeconfig_controller.go +++ b/controllers/sriovnetworknodeconfig_controller.go @@ -1,14 +1,25 @@ package controllers import ( + "bytes" "context" + "fmt" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ) // SriovNetworkNodeConfigReconciler reconciles a SriovNetworkNodeConfig object @@ -32,11 +43,35 @@ type SriovNetworkNodeConfigReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.6.4/pkg/reconcile func (r *SriovNetworkNodeConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { _ = context.Background() - _ = r.Log.WithValues("sriovnetworknodeconfig", req.NamespacedName) + reqLogger := r.Log.WithValues("sriovnetworknodeconfig", req.NamespacedName) - // your logic here + defaultConfig := &sriovnetworkv1.SriovNetworkNodeConfig{} + err := r.Get(context.TODO(), types.NamespacedName{ + Name: DEFAULT_CONFIG_NAME, Namespace: namespace}, defaultConfig) + if err != nil { + if errors.IsNotFound(err) { + // Default Config object not found, create it. + defaultConfig.SetNamespace(namespace) + defaultConfig.SetName(DEFAULT_CONFIG_NAME) + err = r.Create(context.TODO(), defaultConfig) + if err != nil { + reqLogger.Error(err, "Failed to create default Operator Config", "Namespace", + namespace, "Name", DEFAULT_CONFIG_NAME) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } - return ctrl.Result{}, nil + if utils.ClusterType == utils.ClusterTypeOpenshift { + if err = r.syncOvsHardwareOffloadMachineConfig(defaultConfig); err != nil { + return reconcile.Result{}, err + } + } + + return reconcile.Result{RequeueAfter: ResyncPeriod}, nil } // SetupWithManager sets up the controller with the Manager. @@ -45,3 +80,119 @@ func (r *SriovNetworkNodeConfigReconciler) SetupWithManager(mgr ctrl.Manager) er For(&sriovnetworkv1.SriovNetworkNodeConfig{}). Complete(r) } + +func (r *SriovNetworkNodeConfigReconciler) syncOvsHardwareOffloadMachineConfig(dc *sriovnetworkv1.SriovNetworkNodeConfig) error { + logger := r.Log.WithName("syncOvsHardwareOffloadMachineConfig") + + data := render.MakeRenderData() + mcpMap := make(map[string]bool) + nodeOffloadStatus := []sriovnetworkv1.OvsHardwareOffloadConfigStatus{} + + mcpList := &mcfgv1.MachineConfigPoolList{} + err := r.List(context.TODO(), mcpList, &client.ListOptions{}) + if err != nil { + return fmt.Errorf("Failed to get MachineConfigPoolList: %v", err) + } + + for _, mcp := range mcpList.Items { + selector, err := metav1.LabelSelectorAsSelector(mcp.Spec.NodeSelector) + if err != nil { + return fmt.Errorf("Invalid label selector in MachineConfigPool: %s, %v", mcp.GetName(), err) + } + // Node is selected when its label(s) are included in NodeSelector + for _, ovsHWOLConfig := range dc.Spec.OvsHardwareOffload { + if selector.Matches(labels.Set(ovsHWOLConfig.NodeSelector)) { + // OVS Hardware Offload is not supported on master nodes + if mcp.GetName() == "master" { + logger.Info("OVS Hardware Offload is configured on master nodes which is not supported, ignoring.") + continue + } + mcpMap[mcp.GetName()] = true + break + } + } + } + + for mcpName, enable := range mcpMap { + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, mcpName, true, &data) + if err != nil { + return err + } + + foundMC := &mcfgv1.MachineConfig{} + err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) + if err != nil { + if errors.IsNotFound(err) { + if enable { + err = r.Create(context.TODO(), mc) + if err != nil { + return fmt.Errorf("Couldn't create MachineConfig: %v", err) + } + nodeOffloadStatus = append(nodeOffloadStatus, + sriovnetworkv1.OvsHardwareOffloadConfigStatus{Nodes: []string{mcpName}}) + logger.Info("Created MachineConfig CR in MachineConfigPool", mcName, mcpName) + } + } else { + return fmt.Errorf("Failed to get MachineConfig: %v", err) + } + } else { + if enable { + if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { + logger.Info("MachineConfig already exists, updating") + err = r.Update(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't update MachineConfig: %v", err) + } + nodeOffloadStatus = append(nodeOffloadStatus, + sriovnetworkv1.OvsHardwareOffloadConfigStatus{Nodes: []string{mcpName}}) + } else { + logger.Info("No content change, skip updating MC") + } + } else { + logger.Info("offload disabled, delete MachineConfig") + err = r.Delete(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't delete MachineConfig: %v", err) + } + } + } + } + + // Remove legacy MCs + for _, mcp := range mcpList.Items { + found := false + for mcpName := range mcpMap { + if mcp.Name == mcpName { + found = true + break + } + } + if !found { + mcName := "00-" + mcp.Name + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + foundMC := &mcfgv1.MachineConfig{} + err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) + if err != nil { + if !errors.IsNotFound(err) { + err = r.Delete(context.TODO(), foundMC) + if err != nil { + return fmt.Errorf("Couldn't delete MachineConfig: %v", err) + } + } + } + } + } + + // Update SriovOperatorConfig.Status + if equality.Semantic.DeepDerivative(dc.Status.OvsHardwareOffload, nodeOffloadStatus) { + logger.Info("Default SriovOperatorConfig status changed, updating") + dc.Status.OvsHardwareOffload = nodeOffloadStatus + err := r.Status().Update(context.TODO(), dc) + if err != nil { + logger.Error(err, "Fail to update OvsHardwareConfigStatus in default SriovOperatorConfig status") + return err + } + } + + return nil +} diff --git a/controllers/sriovnetworknodeconfig_controller_test.go b/controllers/sriovnetworknodeconfig_controller_test.go new file mode 100644 index 0000000000..6345b84efd --- /dev/null +++ b/controllers/sriovnetworknodeconfig_controller_test.go @@ -0,0 +1,67 @@ +package controllers + +import ( + goctx "context" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + util "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" +) + +var _ = Describe("Operator", func() { + Context("When is up", func() { + It("should be able to create machine config for ovs hardware offload", func() { + config := &sriovnetworkv1.SriovNetworkNodeConfig{} + err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", interval, timeout) + Expect(err).NotTo(HaveOccurred()) + + mcpName := "worker" + mc := &mcfgv1.MachineConfig{} + mcName := "00-" + mcpName + "-" + OVS_HWOL_MACHINE_CONFIG_NAME_SUFFIX + err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp := &mcfgv1.MachineConfigPool{} + err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) + Expect(errors.IsNotFound(err)).Should(BeTrue()) + + mcp = &mcfgv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcpName, + }, + Spec: mcfgv1.MachineConfigPoolSpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/worker": "", + }, + }, + }, + } + err = k8sClient.Create(goctx.TODO(), mcp) + Expect(err).NotTo(HaveOccurred()) + + config.Spec.OvsHardwareOffload = []sriovnetworkv1.OvsHardwareOffloadConfig{ + { + NodeSelector: map[string]string{"node-role.kubernetes.io/worker": ""}, + }, + } + err = k8sClient.Update(goctx.TODO(), config) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() error { + mc := &mcfgv1.MachineConfig{} + err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) + if err != nil { + return err + } + return nil + }, timeout*3, interval).Should(Succeed()) + }) + }) +}) diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index ca123c375f..ab12d7c78d 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -17,14 +17,11 @@ limitations under the License. package controllers import ( - "bytes" "context" "fmt" "os" "github.com/go-logr/logr" - "github.com/openshift/machine-config-operator/lib/resourcemerge" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -112,12 +109,6 @@ func (r *SriovOperatorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result return reconcile.Result{}, err } - if utils.ClusterType == utils.ClusterTypeOpenshift { - if err = r.syncOffloadMachineConfig(defaultConfig); err != nil { - return reconcile.Result{}, err - } - } - return reconcile.Result{RequeueAfter: ResyncPeriod}, nil } @@ -408,102 +399,3 @@ func (r *SriovOperatorConfigReconciler) syncK8sResource(cr *sriovnetworkv1.Sriov } return nil } - -func (r *SriovOperatorConfigReconciler) syncOffloadMachineConfig(dc *sriovnetworkv1.SriovOperatorConfig) error { - logger := r.Log.WithName("syncOffloadMachineConfig") - var err error - - logger.Info("Start to render MachineConfig and MachineConfigPool for OVS HW offloading") - data := render.MakeRenderData() - data.Data["HwOffloadNodeLabel"] = HwOffloadNodeLabel - mcName := "00-" + HwOffloadNodeLabel - mcpName := HwOffloadNodeLabel - mc, err := render.GenerateMachineConfig("bindata/manifests/switchdev-config", mcName, HwOffloadNodeLabel, dc.Spec.EnableOvsOffload, &data) - if err != nil { - return err - } - mcpRaw, err := render.RenderTemplate("bindata/manifests/switchdev-config/machineconfigpool.yaml", &data) - if err != nil { - return err - } - mcp := &mcfgv1.MachineConfigPool{} - if len(mcpRaw) != 1 { - return fmt.Errorf("Invalid MachineConfigPool CR template") - } - err = r.Scheme.Convert(mcpRaw[0], mcp, context.TODO()) - if err != nil { - return err - } - - foundMC := &mcfgv1.MachineConfig{} - foundMCP := &mcfgv1.MachineConfigPool{} - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcName}, foundMC) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mc) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfig: %v", err) - } - logger.Info("Created MachineConfig CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfig: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - if bytes.Compare(foundMC.Spec.Config.Raw, mc.Spec.Config.Raw) == 0 { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MC") - } - } else { - logger.Info("offload disabled, delete MachineConfig") - err = r.Delete(context.TODO(), foundMC) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfig: %v", err) - } - } - } - - err = r.Get(context.TODO(), types.NamespacedName{Name: mcpName}, foundMCP) - if err != nil { - if errors.IsNotFound(err) { - if dc.Spec.EnableOvsOffload { - err = r.Create(context.TODO(), mcp) - if err != nil { - return fmt.Errorf("Couldn't create MachineConfigPool: %v", err) - } - logger.Info("Created MachineConfigPool CR") - } - } else { - return fmt.Errorf("Failed to get MachineConfigPool: %v", err) - } - } else { - if dc.Spec.EnableOvsOffload { - modified := resourcemerge.BoolPtr(false) - resourcemerge.EnsureMachineConfigPool(modified, foundMCP, *mcp) - if *modified { - logger.Info("MachineConfig already exists, updating") - err = r.Update(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't update MachineConfig: %v", err) - } - } else { - logger.Info("No content change, skip updating MCP") - } - } else { - logger.Info("offload disabled, delete MachineConfigPool") - err = r.Delete(context.TODO(), foundMCP) - if err != nil { - return fmt.Errorf("Couldn't delete MachineConfigPool: %v", err) - } - } - } - return nil -} diff --git a/controllers/sriovoperatorconfig_controller_test.go b/controllers/sriovoperatorconfig_controller_test.go index 6b127b277c..b81f8e2d3c 100644 --- a/controllers/sriovoperatorconfig_controller_test.go +++ b/controllers/sriovoperatorconfig_controller_test.go @@ -5,13 +5,11 @@ import ( admv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" util "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" @@ -174,64 +172,5 @@ var _ = Describe("Operator", func() { }, timeout*10, interval).Should(Equal(config.Spec.ConfigDaemonNodeSelector)) }) - It("should be able to create/remove hw offload MCO and MC", func() { - config := &sriovnetworkv1.SriovOperatorConfig{} - err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", interval, timeout) - Expect(err).NotTo(HaveOccurred()) - By("by default") - mcName := "00-" + HwOffloadNodeLabel - mc := &mcfgv1.MachineConfig{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - mcpName := HwOffloadNodeLabel - mcp := &mcfgv1.MachineConfigPool{} - err = k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - Expect(errors.IsNotFound(err)).Should(BeTrue()) - - By("set EnableOvsOffload to true to create MC and MCP resources") - config.Spec.EnableOvsOffload = true - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - Eventually(func() error { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return err - } - return nil - }, timeout*3, interval).Should(Succeed()) - - By("set EnableOvsOffload to false to delete MC and MCP resources") - config.Spec.EnableOvsOffload = false - err = k8sClient.Update(goctx.TODO(), config) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() bool { - mc := &mcfgv1.MachineConfig{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcName, Namespace: testNamespace}, mc) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - Eventually(func() bool { - mcp := &mcfgv1.MachineConfigPool{} - err := k8sClient.Get(goctx.TODO(), types.NamespacedName{Name: mcpName, Namespace: testNamespace}, mcp) - if err != nil { - return errors.IsNotFound(err) - } - return false - }, timeout*3, interval).Should(BeTrue()) - }) }) }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index d136fc546d..2f5ad3ca94 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -121,6 +121,13 @@ var _ = BeforeSuite(func(done Done) { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&SriovNetworkNodeConfigReconciler{ + Client: k8sManager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("SriovNetworkNodeConfig"), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + os.Setenv("RESOURCE_PREFIX", "openshift.io") os.Setenv("NAMESPACE", "openshift-sriov-network-operator") os.Setenv("ENABLE_ADMISSION_CONTROLLER", "true") @@ -159,6 +166,12 @@ var _ = BeforeSuite(func(done Done) { LogLevel: 2, } Expect(k8sClient.Create(context.TODO(), config)).Should(Succeed()) + + nodeConfig := &sriovnetworkv1.SriovNetworkNodeConfig{} + nodeConfig.SetNamespace(testNamespace) + nodeConfig.SetName(DEFAULT_CONFIG_NAME) + nodeConfig.Spec = sriovnetworkv1.SriovNetworkNodeConfigSpec{} + Expect(k8sClient.Create(context.TODO(), nodeConfig)).Should(Succeed()) close(done) }, 60) diff --git a/main.go b/main.go index c77906a75a..7b55265b76 100644 --- a/main.go +++ b/main.go @@ -142,6 +142,13 @@ func main() { os.Exit(1) } + // Create default SriovNetworkNodeConfig + err = createDefaultNodeConfig(ctrl.GetConfigOrDie()) + if err != nil { + setupLog.Error(err, "unable to create default SriovNetworkNodeConfig") + os.Exit(1) + } + stopCh := ctrl.SetupSignalHandler() go func() { if err := mgrGlobal.Start(stopCh); err != nil { @@ -222,3 +229,31 @@ func createDefaultOperatorConfig(cfg *rest.Config) error { } return nil } + +func createDefaultNodeConfig(cfg *rest.Config) error { + logger := setupLog.WithName("createDefaultNodeConfig") + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("Couldn't create client: %v", err) + } + config := &sriovnetworkv1.SriovNetworkNodeConfig{ + Spec: sriovnetworkv1.SriovNetworkNodeConfigSpec{}, + } + name := "default" + namespace := os.Getenv("NAMESPACE") + err = c.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, config) + if err != nil { + if errors.IsNotFound(err) { + logger.Info("Create default SriovNetworkNodeConfig") + config.Namespace = namespace + config.Name = name + err = c.Create(context.TODO(), config) + if err != nil { + return err + } + } + // Error reading the object - requeue the request. + return err + } + return nil +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index b01d90facc..65925e68f6 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -489,7 +489,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { reqReboot := false reqDrain := false - mcoReboot := false for k, p := range dn.LoadedPlugins { d, r := false, false if dn.nodeState.GetName() == "" { @@ -501,11 +500,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.Errorf("nodeStateSyncHandler(): plugin %s error: %v", k, err) return err } - - if p.Name() == "mco_plugin" && d && !r { - mcoReboot = true - } - glog.V(0).Infof("nodeStateSyncHandler(): plugin %s: reqDrain %v, reqReboot %v", k, d, r) reqDrain = reqDrain || d reqReboot = reqReboot || r @@ -513,7 +507,7 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { glog.V(0).Infof("nodeStateSyncHandler(): reqDrain %v, reqReboot %v disableDrain %v", reqDrain, reqReboot, dn.disableDrain) for k, p := range dn.LoadedPlugins { - if k != GenericPlugin && k != McoPlugin { + if k != GenericPlugin { err := p.Apply() if err != nil { glog.Errorf("nodeStateSyncHandler(): plugin %s fail to apply: %v", k, err) @@ -521,15 +515,6 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } } } - if mcoReboot { - glog.V(0).Infof("nodeStateSyncHandler(): MCO will trigger reboot") - err = dn.LoadedPlugins[McoPlugin].Apply() - if err != nil { - glog.Errorf("nodeStateSyncHandler(): mco_plugin fail to apply: %v", err) - return err - } - return nil - } if utils.ClusterType == utils.ClusterTypeOpenshift { if err = dn.getNodeMachinePool(); err != nil { return err @@ -674,9 +659,7 @@ func (dn *Daemon) loadVendorPlugins(ns *sriovnetworkv1.SriovNetworkNodeState) er pl = append(pl, VirtualPlugin) } else { pl = registerPlugins(ns) - if utils.ClusterType == utils.ClusterTypeOpenshift { - pl = append(pl, McoPlugin) - } else { + if utils.ClusterType != utils.ClusterTypeOpenshift { pl = append(pl, K8sPlugin) } pl = append(pl, GenericPlugin) diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index ffab8607d9..eb4458f1bd 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -30,7 +30,6 @@ const ( SpecVersion = "1.0" GenericPlugin = "generic_plugin" VirtualPlugin = "virtual_plugin" - McoPlugin = "mco_plugin" K8sPlugin = "k8s_plugin" ) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 966fdc8e0c..8429565383 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -59,18 +59,10 @@ func (p *GenericPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeSta p.DesireState = state needDrain = needDrainNode(state.Spec.Interfaces, state.Status.Interfaces) + needReboot = needRebootNode(state, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(state) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -84,18 +76,10 @@ func (p *GenericPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkN p.DesireState = new needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces) + needReboot = needRebootNode(new, &p.LoadVfioDriver) - if p.LoadVfioDriver != loaded { - if needVfioDriver(new) { - p.LoadVfioDriver = loading - if needReboot, err = tryEnableIommuInKernelArgs(); err != nil { - glog.Errorf("generic-plugin OnNodeStateAdd():fail to enable iommu in kernel args: %v", err) - return - } - } - if needReboot { - needDrain = true - } + if needReboot { + needDrain = true } return } @@ -212,3 +196,31 @@ func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.Int } return } + +func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint) (needReboot bool) { + needReboot = false + if *loadVfioDriver != loaded { + if needVfioDriver(state) { + *loadVfioDriver = loading + update, err := tryEnableIommuInKernelArgs() + if err != nil { + glog.Errorf("generic-plugin needRebootNode():fail to enable iommu in kernel args: %v", err) + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for enabling iommu kernel args") + } + needReboot = needReboot || update + } + } + + update, err := utils.WriteSwitchdevConfFile(state) + if err != nil { + glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file") + } + if update { + glog.V(2).Infof("generic-plugin needRebootNode(): need reboot for updating switchdev device configuration") + } + needReboot = needReboot || update + + return +} diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index c26d82b42f..e3def1e000 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -117,34 +117,18 @@ func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS p.updateTarget.reset() // TODO add check for enableOvsOffload in OperatorConfig later - // Update services if switchdev not required - if utils.IsSwitchdevModeSpec(new.Spec) { - // Check services - err = p.servicesStateUpdate() - if err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) - return - } - } - - // Check switchdev config - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) + // Update services if switchdev required + if !utils.IsSwitchdevModeSpec(new.Spec) { return } - if remove { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to clean switchdev VFs") - needDrain = true - needReboot = true - return - } - if update { - glog.Info("k8s-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true + + // Check services + err = p.servicesStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) return } + if p.updateTarget.needUpdate() { needDrain = true if p.updateTarget.switchdevUdevScript { diff --git a/pkg/plugins/mco/mco_plugin.go b/pkg/plugins/mco/mco_plugin.go deleted file mode 100644 index b63c8dfe31..0000000000 --- a/pkg/plugins/mco/mco_plugin.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - - "github.com/golang/glog" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -type McoPlugin struct { - PluginName string - SpecVersion string - DesireState *sriovnetworkv1.SriovNetworkNodeState - LastState *sriovnetworkv1.SriovNetworkNodeState -} - -const ( - switchdevUnitPath = "/host/etc/systemd/system/switchdev-configuration.service" - nodeLabelPrefix = "node-role.kubernetes.io/" -) - -var nodeName string -var Plugin McoPlugin -var kubeclient *kubernetes.Clientset -var switchdevConfigured bool - -// Initialize our plugin and set up initial values -func init() { - Plugin = McoPlugin{ - PluginName: "mco_plugin", - SpecVersion: "1.0", - } - - var config *rest.Config - var err error - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig != "" { - config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - } else { - // creates the in-cluster config - config, err = rest.InClusterConfig() - } - - if err != nil { - panic(err.Error()) - } - kubeclient = kubernetes.NewForConfigOrDie(config) -} - -// Name returns the name of the plugin -func (p *McoPlugin) Name() string { - return p.PluginName -} - -// Spec returns the version of the spec expected by the plugin -func (p *McoPlugin) Spec() string { - return p.SpecVersion -} - -// OnNodeStateAdd Invoked when SriovNetworkNodeState CR is created, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateAdd(state *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateAdd()") - nodeName = state.GetName() - return p.OnNodeStateChange(nil, state) -} - -// OnNodeStateChange Invoked when SriovNetworkNodeState CR is updated, return if need dain and/or reboot node -func (p *McoPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - glog.Info("mco-plugin OnNodeStateChange()") - switchdevConfigured = utils.IsSwitchdevModeSpec(new.Spec) - - var update, remove bool - if update, remove, err = utils.WriteSwitchdevConfFile(new); err != nil { - glog.Errorf("mco-plugin OnNodeStateChange():fail to update switchdev.conf file: %v", err) - return - } - if remove { - needDrain = true - return - } - if update { - if _, err = os.Stat(switchdevUnitPath); err != nil { - if os.IsNotExist(err) { - glog.Info("mco-plugin OnNodeStateChange(): the latest MachineConfig has not been applied") - needDrain = true - err = nil - return - } - return - } - // node is already in the offload MCP - glog.Info("mco-plugin OnNodeStateChange(): need reboot node to use the up-to-date switchdev.conf") - needDrain = true - needReboot = true - return - } - return -} - -// Apply config change -func (p *McoPlugin) Apply() error { - glog.Info("mco-plugin Apply()") - node, err := kubeclient.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - label := nodeLabelPrefix + controllers.HwOffloadNodeLabel - if switchdevConfigured { - if _, ok := node.Labels[label]; !ok { - glog.Infof("Move node %s into HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: "", - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is already in HW offload MachineConfigPool", node.Name) - return nil - } - - if _, ok := node.Labels[label]; ok { - glog.Infof("Remove node %s from HW offload MachineConfigPool", node.Name) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - label: nil, - }, - }, - }) - kubeclient.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, mergePatch, metav1.PatchOptions{}) - return nil - } - glog.Infof("Node %s is not in HW offload MachineConfigPool", node.Name) - return nil -} diff --git a/pkg/utils/switchdev.go b/pkg/utils/switchdev.go index 26d7819a07..c94373178b 100644 --- a/pkg/utils/switchdev.go +++ b/pkg/utils/switchdev.go @@ -23,7 +23,7 @@ func IsSwitchdevModeSpec(spec sriovnetworkv1.SriovNetworkNodeStateSpec) bool { return false } -func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update, remove bool, err error) { +func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (update bool, err error) { _, err = os.Stat(switchDevConfPath) if err != nil { if os.IsNotExist(err) { @@ -52,10 +52,6 @@ func WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (upd glog.V(2).Info("WriteSwitchdevConfFile(): no update") return } - if newContent == "" { - remove = true - glog.V(2).Info("WriteSwitchdevConfFile(): remove content in switchdev.conf") - } update = true glog.V(2).Infof("WriteSwitchdevConfFile(): write %s to switchdev.conf", newContent) err = ioutil.WriteFile(switchDevConfPath, []byte(newContent), 0644)