From f44285ad9c78db5e6afcf42b35c2ddfdd1fb5a83 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Tue, 7 Dec 2021 12:22:06 +0200 Subject: [PATCH 1/3] Support pause MCP on SNO This commit allow to still pause the MCP when running on OCP. This is needed when you have only 1 node in the cluster and the drainSkip is true Signed-off-by: Sebastian Sch --- pkg/daemon/daemon.go | 186 ++++++++++++++++++++++++------------------- 1 file changed, 104 insertions(+), 82 deletions(-) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 6a9f5c1aa..138b3fb6a 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -497,11 +497,26 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } } - if reqDrain && !dn.disableDrain { - glog.Info("nodeStateSyncHandler(): drain node") - if err := dn.drainNode(dn.name); err != nil { + if reqDrain { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + glog.Infof("nodeStateSyncHandler(): get drain lock for sriov daemon") + done := make(chan bool) + go dn.getDrainLock(ctx, done) + <-done + + glog.Infof("nodeStateSyncHandler(): pause MCP") + if err := dn.pauseMCP(); err != nil { return err } + + if !dn.disableDrain { + glog.Info("nodeStateSyncHandler(): drain node") + if err := dn.drainNode(); err != nil { + return err + } + } } if !reqReboot { @@ -551,8 +566,10 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } func (dn *Daemon) completeDrain() error { - if err := drain.RunCordonOrUncordon(dn.drainer, dn.node, false); err != nil { - return err + if !dn.disableDrain { + if err := drain.RunCordonOrUncordon(dn.drainer, dn.node, false); err != nil { + return err + } } if utils.ClusterType == utils.ClusterTypeOpenshift { @@ -732,7 +749,7 @@ func (dn *Daemon) annotateNode(node, value string) error { func (dn *Daemon) getNodeMachinePool() error { desiredConfig, ok := dn.node.Annotations[daemonconsts.DesiredMachineConfigAnnotationKey] if !ok { - glog.Error("getNodeMachinePool(): Failed to find the the desiredConfig Annotation") + glog.Errorf("getNodeMachinePool(): Failed to find the the desiredConfig Annotation") return fmt.Errorf("getNodeMachinePool(): Failed to find the the desiredConfig Annotation") } mc, err := dn.mcClient.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), desiredConfig, metav1.GetOptions{}) @@ -801,99 +818,104 @@ func (dn *Daemon) getDrainLock(ctx context.Context, done chan bool) { }) } -func (dn *Daemon) drainNode(name string) error { - glog.Info("drainNode(): Update prepared") +func (dn *Daemon) pauseMCP() error { + glog.Info("pauseMCP(): check if pausing MCP is possible") var err error - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - done := make(chan bool) - go dn.getDrainLock(ctx, done) - <-done + if utils.ClusterType != utils.ClusterTypeOpenshift { + glog.Infof("pauseMCP(): skipping MCP pause as the cluster is not an openshift cluster") + return nil + } - if utils.ClusterType == utils.ClusterTypeOpenshift { - mcpInformerFactory := mcfginformers.NewSharedInformerFactory(dn.mcClient, - time.Second*30, - ) - mcpInformer := mcpInformerFactory.Machineconfiguration().V1().MachineConfigPools().Informer() + mcpInformerFactory := mcfginformers.NewSharedInformerFactory(dn.mcClient, + time.Second*30, + ) + mcpInformer := mcpInformerFactory.Machineconfiguration().V1().MachineConfigPools().Informer() - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - paused := dn.node.Annotations[annoKey] == annoMcpPaused + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + paused := dn.node.Annotations[annoKey] == annoMcpPaused - mcpEventHandler := func(obj interface{}) { - mcp := obj.(*mcfgv1.MachineConfigPool) - if mcp.GetName() != dn.mcpName { + mcpEventHandler := func(obj interface{}) { + mcp := obj.(*mcfgv1.MachineConfigPool) + if mcp.GetName() != dn.mcpName { + return + } + // Always get the latest object + newMcp := &mcfgv1.MachineConfigPool{} + newMcp, err = dn.mcClient.MachineconfigurationV1().MachineConfigPools().Get(ctx, dn.mcpName, metav1.GetOptions{}) + if err != nil { + glog.V(2).Infof("pauseMCP(): Failed to get MCP %s: %v", dn.mcpName, err) + return + } + if mcfgv1.IsMachineConfigPoolConditionFalse(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolDegraded) && + mcfgv1.IsMachineConfigPoolConditionTrue(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated) && + mcfgv1.IsMachineConfigPoolConditionFalse(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdating) { + glog.V(2).Infof("pauseMCP(): MCP %s is ready", dn.mcpName) + if paused { + glog.V(2).Info("pauseMCP(): stop MCP informer") + cancel() return } - // Always get the latest object - newMcp, err := dn.mcClient.MachineconfigurationV1().MachineConfigPools().Get(ctx, dn.mcpName, metav1.GetOptions{}) + if newMcp.Spec.Paused { + glog.V(2).Infof("pauseMCP(): MCP %s was paused by other, wait...", dn.mcpName) + return + } + glog.Infof("pauseMCP(): pause MCP %s", dn.mcpName) + pausePatch := []byte("{\"spec\":{\"paused\":true}}") + _, err = dn.mcClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) if err != nil { - glog.V(2).Infof("drainNode(): Failed to get MCP %s: %v", dn.mcpName, err) + glog.V(2).Infof("pauseMCP(): Failed to pause MCP %s: %v", dn.mcpName, err) return } - if mcfgv1.IsMachineConfigPoolConditionFalse(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolDegraded) && - mcfgv1.IsMachineConfigPoolConditionTrue(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated) && - mcfgv1.IsMachineConfigPoolConditionFalse(newMcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdating) { - glog.V(2).Infof("drainNode(): MCP %s is ready", dn.mcpName) - if paused { - glog.V(2).Info("drainNode(): stop MCP informer") - cancel() - return - } - if newMcp.Spec.Paused { - glog.V(2).Infof("drainNode(): MCP %s was paused by other, wait...", dn.mcpName) - return - } - glog.Infof("drainNode(): pause MCP %s", dn.mcpName) - pausePatch := []byte("{\"spec\":{\"paused\":true}}") - _, err = dn.mcClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) - if err != nil { - glog.V(2).Infof("drainNode(): Failed to pause MCP %s: %v", dn.mcpName, err) - return - } - err = dn.annotateNode(dn.name, annoMcpPaused) - if err != nil { - glog.V(2).Infof("drainNode(): Failed to annotate node: %v", err) - return - } - paused = true + err = dn.annotateNode(dn.name, annoMcpPaused) + if err != nil { + glog.V(2).Infof("pauseMCP(): Failed to annotate node: %v", err) return } - if paused { - glog.Infof("drainNode(): MCP is processing, resume MCP %s", dn.mcpName) - pausePatch := []byte("{\"spec\":{\"paused\":false}}") - _, err = dn.mcClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) - if err != nil { - glog.V(2).Infof("drainNode(): fail to resume MCP %s: %v", dn.mcpName, err) - return - } - err = dn.annotateNode(dn.name, annoDraining) - if err != nil { - glog.V(2).Infof("drainNode(): Failed to annotate node: %v", err) - return - } - paused = false + paused = true + return + } + if paused { + glog.Infof("pauseMCP(): MCP is processing, resume MCP %s", dn.mcpName) + pausePatch := []byte("{\"spec\":{\"paused\":false}}") + _, err = dn.mcClient.MachineconfigurationV1().MachineConfigPools().Patch(context.Background(), dn.mcpName, types.MergePatchType, pausePatch, metav1.PatchOptions{}) + if err != nil { + glog.V(2).Infof("pauseMCP(): fail to resume MCP %s: %v", dn.mcpName, err) + return } - glog.Infof("drainNode():MCP %s is not ready: %v, wait...", newMcp.GetName(), newMcp.Status.Conditions) + err = dn.annotateNode(dn.name, annoDraining) + if err != nil { + glog.V(2).Infof("pauseMCP(): Failed to annotate node: %v", err) + return + } + paused = false } + glog.Infof("pauseMCP():MCP %s is not ready: %v, wait...", newMcp.GetName(), newMcp.Status.Conditions) + } - mcpInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: mcpEventHandler, - UpdateFunc: func(old, new interface{}) { - mcpEventHandler(new) - }, - }) - - // The Draining_MCP_Paused state means the MCP work has been paused by the config daemon in previous round. - // Only check MCP state if the node is not in Draining_MCP_Paused state - if !paused { - mcpInformerFactory.Start(ctx.Done()) - mcpInformerFactory.WaitForCacheSync(ctx.Done()) - <-ctx.Done() - } + mcpInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: mcpEventHandler, + UpdateFunc: func(old, new interface{}) { + mcpEventHandler(new) + }, + }) + + // The Draining_MCP_Paused state means the MCP work has been paused by the config daemon in previous round. + // Only check MCP state if the node is not in Draining_MCP_Paused state + if !paused { + mcpInformerFactory.Start(ctx.Done()) + mcpInformerFactory.WaitForCacheSync(ctx.Done()) + <-ctx.Done() } + return err +} + +func (dn *Daemon) drainNode() error { + glog.Info("drainNode(): Update prepared") + var err error + backoff := wait.Backoff{ Steps: 5, Duration: 10 * time.Second, From a4910597cdd782578dba992a6ef64d0b53c7c9a2 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Tue, 7 Dec 2021 20:21:27 +0200 Subject: [PATCH 2/3] Add the DisableDrain when running one a single node Signed-off-by: Sebastian Sch --- controllers/sriovoperatorconfig_controller.go | 7 ++++ main.go | 7 ++++ pkg/daemon/daemon.go | 40 ++++++++++--------- pkg/utils/cluster.go | 26 ++++++++++++ 4 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 pkg/utils/cluster.go diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index c47f29ef4..92e10075c 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -78,6 +78,11 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. Name: constants.DEFAULT_CONFIG_NAME, Namespace: namespace}, defaultConfig) if err != nil { if errors.IsNotFound(err) { + singleNode, err := utils.IsSingleNodeCluster(r.Client) + if err != nil { + return reconcile.Result{}, fmt.Errorf("Couldn't check the anount of nodes in the cluster") + } + // Default Config object not found, create it. defaultConfig.SetNamespace(namespace) defaultConfig.SetName(constants.DEFAULT_CONFIG_NAME) @@ -86,7 +91,9 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. EnableOperatorWebhook: func() *bool { b := enableAdmissionController; return &b }(), ConfigDaemonNodeSelector: map[string]string{}, LogLevel: 2, + DisableDrain: singleNode, } + err = r.Create(context.TODO(), defaultConfig) if err != nil { logger.Error(err, "Failed to create default Operator Config", "Namespace", diff --git a/main.go b/main.go index 63a84bac1..3050cf95e 100644 --- a/main.go +++ b/main.go @@ -232,6 +232,12 @@ func createDefaultOperatorConfig(cfg *rest.Config) error { if err != nil { return fmt.Errorf("Couldn't create client: %v", err) } + + singleNode, err := utils.IsSingleNodeCluster(c) + if err != nil { + return fmt.Errorf("Couldn't check the anount of nodes in the cluster") + } + enableAdmissionController := os.Getenv("ENABLE_ADMISSION_CONTROLLER") == "true" config := &sriovnetworkv1.SriovOperatorConfig{ Spec: sriovnetworkv1.SriovOperatorConfigSpec{ @@ -239,6 +245,7 @@ func createDefaultOperatorConfig(cfg *rest.Config) error { EnableOperatorWebhook: func() *bool { b := enableAdmissionController; return &b }(), ConfigDaemonNodeSelector: map[string]string{}, LogLevel: 2, + DisableDrain: singleNode, }, } name := "default" diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 138b3fb6a..b3efc9871 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -498,25 +498,27 @@ func (dn *Daemon) nodeStateSyncHandler(generation int64) error { } if reqDrain { - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - glog.Infof("nodeStateSyncHandler(): get drain lock for sriov daemon") - done := make(chan bool) - go dn.getDrainLock(ctx, done) - <-done + if !dn.disableDrain { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() - glog.Infof("nodeStateSyncHandler(): pause MCP") - if err := dn.pauseMCP(); err != nil { - return err + glog.Infof("nodeStateSyncHandler(): get drain lock for sriov daemon") + done := make(chan bool) + go dn.getDrainLock(ctx, done) + <-done } - if !dn.disableDrain { - glog.Info("nodeStateSyncHandler(): drain node") - if err := dn.drainNode(); err != nil { + if utils.ClusterType == utils.ClusterTypeOpenshift { + glog.Infof("nodeStateSyncHandler(): pause MCP") + if err := dn.pauseMCP(); err != nil { return err } } + + glog.Info("nodeStateSyncHandler(): drain node") + if err := dn.drainNode(); err != nil { + return err + } } if !reqReboot { @@ -819,14 +821,9 @@ func (dn *Daemon) getDrainLock(ctx context.Context, done chan bool) { } func (dn *Daemon) pauseMCP() error { - glog.Info("pauseMCP(): check if pausing MCP is possible") + glog.Info("pauseMCP(): pausing MCP") var err error - if utils.ClusterType != utils.ClusterTypeOpenshift { - glog.Infof("pauseMCP(): skipping MCP pause as the cluster is not an openshift cluster") - return nil - } - mcpInformerFactory := mcfginformers.NewSharedInformerFactory(dn.mcClient, time.Second*30, ) @@ -913,6 +910,11 @@ func (dn *Daemon) pauseMCP() error { } func (dn *Daemon) drainNode() error { + if dn.disableDrain { + glog.Info("drainNode(): disable drain is true skipping drain") + return nil + } + glog.Info("drainNode(): Update prepared") var err error diff --git a/pkg/utils/cluster.go b/pkg/utils/cluster.go new file mode 100644 index 000000000..1421e1e62 --- /dev/null +++ b/pkg/utils/cluster.go @@ -0,0 +1,26 @@ +package utils + +import ( + "context" + + "github.com/golang/glog" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func IsSingleNodeCluster(c client.Client) (bool, error) { + nodeList := &corev1.NodeList{} + err := c.List(context.TODO(), nodeList) + if err != nil { + glog.Errorf("IsSingleNodeCluster(): Failed to list nodes: %v", err) + return false, err + } + + if len(nodeList.Items) == 1 { + glog.Infof("IsSingleNodeCluster(): one node found in the cluster") + return true, nil + } + + return false, nil +} From 66f3ae4710a0dbadf0070c82f5590905ed8de4b9 Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Mon, 20 Dec 2021 10:06:26 +0800 Subject: [PATCH 3/3] Add switchdev-after-networkmanager services to k8s plugin Signed-off-by: Zenghui Shi --- ... switchdev-configuration-after-nm.sh.yaml} | 0 ...switchdev-configuration-before-nm.sh.yaml} | 0 ... => switchdev-configuration-after-nm.yaml} | 0 ...=> switchdev-configuration-before-nm.yaml} | 0 pkg/plugins/k8s/k8s_plugin.go | 192 ++++++++++++------ 5 files changed, 128 insertions(+), 64 deletions(-) rename bindata/manifests/switchdev-config/files/{switchdev-configuration-after-NM.sh.yaml => switchdev-configuration-after-nm.sh.yaml} (100%) rename bindata/manifests/switchdev-config/files/{switchdev-configuration-before-NM.sh.yaml => switchdev-configuration-before-nm.sh.yaml} (100%) rename bindata/manifests/switchdev-config/switchdev-units/{switchdev-configuration-after-NM.yaml => switchdev-configuration-after-nm.yaml} (100%) rename bindata/manifests/switchdev-config/switchdev-units/{switchdev-configuration-before-NM.yaml => switchdev-configuration-before-nm.yaml} (100%) diff --git a/bindata/manifests/switchdev-config/files/switchdev-configuration-after-NM.sh.yaml b/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml similarity index 100% rename from bindata/manifests/switchdev-config/files/switchdev-configuration-after-NM.sh.yaml rename to bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml diff --git a/bindata/manifests/switchdev-config/files/switchdev-configuration-before-NM.sh.yaml b/bindata/manifests/switchdev-config/files/switchdev-configuration-before-nm.sh.yaml similarity index 100% rename from bindata/manifests/switchdev-config/files/switchdev-configuration-before-NM.sh.yaml rename to bindata/manifests/switchdev-config/files/switchdev-configuration-before-nm.sh.yaml diff --git a/bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-after-NM.yaml b/bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-after-nm.yaml similarity index 100% rename from bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-after-NM.yaml rename to bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-after-nm.yaml diff --git a/bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-before-NM.yaml b/bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-before-nm.yaml similarity index 100% rename from bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-before-NM.yaml rename to bindata/manifests/switchdev-config/switchdev-units/switchdev-configuration-before-nm.yaml diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index e3def1e00..4db2644bc 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -16,40 +16,50 @@ import ( ) type K8sPlugin struct { - PluginName string - SpecVersion string - serviceManager service.ServiceManager - switchdevRunScript *service.ScriptManifestFile - switchdevUdevScript *service.ScriptManifestFile - switchdevService *service.Service - openVSwitchService *service.Service - networkManagerService *service.Service - updateTarget *k8sUpdateTarget + PluginName string + SpecVersion string + serviceManager service.ServiceManager + switchdevBeforeNMRunScript *service.ScriptManifestFile + switchdevAfterNMRunScript *service.ScriptManifestFile + switchdevUdevScript *service.ScriptManifestFile + switchdevBeforeNMService *service.Service + switchdevAfterNMService *service.Service + openVSwitchService *service.Service + networkManagerService *service.Service + updateTarget *k8sUpdateTarget } type k8sUpdateTarget struct { - switchdevService bool - switchdevRunScript bool - switchdevUdevScript bool - systemServices []*service.Service + switchdevBeforeNMService bool + switchdevAfterNMService bool + switchdevBeforeNMRunScript bool + switchdevAfterNMRunScript bool + switchdevUdevScript bool + systemServices []*service.Service } func (u *k8sUpdateTarget) needUpdate() bool { - return u.switchdevService || u.switchdevRunScript || u.switchdevUdevScript || len(u.systemServices) > 0 + return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || len(u.systemServices) > 0 +} + +func (u *k8sUpdateTarget) needReboot() bool { + return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript } func (u *k8sUpdateTarget) reset() { - u.switchdevService = false - u.switchdevRunScript = false + u.switchdevBeforeNMService = false + u.switchdevAfterNMService = false + u.switchdevBeforeNMRunScript = false + u.switchdevAfterNMRunScript = false u.systemServices = []*service.Service{} } func (u *k8sUpdateTarget) String() string { var updateList []string - if u.switchdevService { + if u.switchdevBeforeNMService || u.switchdevAfterNMService { updateList = append(updateList, "SwitchdevService") } - if u.switchdevRunScript { + if u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript { updateList = append(updateList, "SwitchdevRunScript") } if u.switchdevUdevScript { @@ -63,13 +73,15 @@ func (u *k8sUpdateTarget) String() string { } const ( - switchdevManifestPath = "bindata/manifests/switchdev-config/" - switchdevUnits = switchdevManifestPath + "switchdev-units/" - switchdevUnitFile = switchdevUnits + "switchdev-configuration.yaml" - networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" - ovsUnitFile = switchdevManifestPath + "ovs-units/ovs-vswitchd.service.yaml" - configuresSwitchdevScript = switchdevManifestPath + "files/configure-switchdev.sh.yaml" - switchdevRenamingUdevScript = switchdevManifestPath + "files/switchdev-vf-link-name.sh.yaml" + switchdevManifestPath = "bindata/manifests/switchdev-config/" + switchdevUnits = switchdevManifestPath + "switchdev-units/" + switchdevBeforeNMUnitFile = switchdevUnits + "switchdev-configuration-before-nm.yaml" + switchdevAfterNMUnitFile = switchdevUnits + "switchdev-configuration-after-nm.yaml" + networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" + ovsUnitFile = switchdevManifestPath + "ovs-units/ovs-vswitchd.service.yaml" + configuresSwitchdevBeforeNMScript = switchdevManifestPath + "files/switchdev-configuration-before-nm.sh.yaml" + configuresSwitchdevAfterNMScript = switchdevManifestPath + "files/switchdev-configuration-after-nm.sh.yaml" + switchdevRenamingUdevScript = switchdevManifestPath + "files/switchdev-vf-link-name.sh.yaml" chroot = "/host" ) @@ -131,7 +143,7 @@ func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS if p.updateTarget.needUpdate() { needDrain = true - if p.updateTarget.switchdevUdevScript { + if p.updateTarget.needReboot() { needReboot = true glog.Infof("k8s-plugin OnNodeStateChange(): needReboot to update %q", p.updateTarget) } else { @@ -145,7 +157,7 @@ func (p *K8sPlugin) OnNodeStateChange(old, new *sriovnetworkv1.SriovNetworkNodeS // Apply config change func (p *K8sPlugin) Apply() error { glog.Info("k8s-plugin Apply()") - if err := p.updateSwichdevService(); err != nil { + if err := p.updateSwitchdevService(); err != nil { return err } @@ -160,7 +172,11 @@ func (p *K8sPlugin) Apply() error { func (p *K8sPlugin) readSwitchdevManifest() error { // Read switchdev service - switchdevService, err := service.ReadServiceManifestFile(switchdevUnitFile) + switchdevBeforeNMService, err := service.ReadServiceManifestFile(switchdevBeforeNMUnitFile) + if err != nil { + return err + } + switchdevAfterNMService, err := service.ReadServiceManifestFile(switchdevAfterNMUnitFile) if err != nil { return err } @@ -171,18 +187,28 @@ func (p *K8sPlugin) readSwitchdevManifest() error { Name: "ConditionPathExists", Value: "!/etc/ignition-machine-config-encapsulated.json", } - switchdevService, err = service.RemoveFromService(switchdevService, conditionOpt) + switchdevBeforeNMService, err = service.RemoveFromService(switchdevBeforeNMService, conditionOpt) + if err != nil { + return err + } + switchdevAfterNMService, err = service.RemoveFromService(switchdevAfterNMService, conditionOpt) if err != nil { return err } - p.switchdevService = switchdevService + p.switchdevBeforeNMService = switchdevBeforeNMService + p.switchdevAfterNMService = switchdevAfterNMService // Read switchdev run script - switchdevRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevScript) + switchdevBeforeNMRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevBeforeNMScript) if err != nil { return err } - p.switchdevRunScript = switchdevRunScript + switchdevAfterNMRunScript, err := service.ReadScriptManifestFile(configuresSwitchdevAfterNMScript) + if err != nil { + return err + } + p.switchdevBeforeNMRunScript = switchdevBeforeNMRunScript + p.switchdevAfterNMRunScript = switchdevAfterNMRunScript // Read switchdev udev script switchdevUdevScript, err := service.ReadScriptManifestFile(switchdevRenamingUdevScript) @@ -232,42 +258,35 @@ func (p *K8sPlugin) readManifestFiles() error { func (p *K8sPlugin) switchdevServiceStateUpdate() error { // Check switchdev service - swdService, err := p.serviceManager.ReadService(p.switchdevService.Path) + needUpdate, err := p.isSwitchdevServiceNeedUpdate(p.switchdevBeforeNMService) if err != nil { - if !os.IsNotExist(err) { - return err - } - // service not exists - p.updateTarget.switchdevService = true - } else { - needChange, err := service.CompareServices(swdService, p.switchdevService) - if err != nil { - return err - } - p.updateTarget.switchdevService = needChange + return err + } + p.updateTarget.switchdevBeforeNMService = needUpdate + needUpdate, err = p.isSwitchdevServiceNeedUpdate(p.switchdevAfterNMService) + if err != nil { + return err } + p.updateTarget.switchdevAfterNMService = needUpdate // Check switchdev run script - data, err := ioutil.ReadFile(path.Join(chroot, p.switchdevRunScript.Path)) + needUpdate, err = p.isSwitchdevScriptNeedUpdate(p.switchdevBeforeNMRunScript) if err != nil { - if !os.IsNotExist(err) { - return err - } - p.updateTarget.switchdevRunScript = true - } else if string(data) != p.switchdevRunScript.Contents.Inline { - p.updateTarget.switchdevRunScript = true + return err + } + p.updateTarget.switchdevBeforeNMRunScript = needUpdate + needUpdate, err = p.isSwitchdevScriptNeedUpdate(p.switchdevAfterNMRunScript) + if err != nil { + return err } + p.updateTarget.switchdevAfterNMRunScript = needUpdate // Check switchdev udev script - data, err = ioutil.ReadFile(path.Join(chroot, p.switchdevUdevScript.Path)) + needUpdate, err = p.isSwitchdevScriptNeedUpdate(p.switchdevUdevScript) if err != nil { - if !os.IsNotExist(err) { - return err - } - p.updateTarget.switchdevUdevScript = true - } else if string(data) != p.switchdevUdevScript.Contents.Inline { - p.updateTarget.switchdevUdevScript = true + return err } + p.updateTarget.switchdevUdevScript = needUpdate return nil } @@ -276,6 +295,36 @@ func (p *K8sPlugin) getSystemServices() []*service.Service { return []*service.Service{p.networkManagerService, p.openVSwitchService} } +func (p *K8sPlugin) isSwitchdevScriptNeedUpdate(scriptObj *service.ScriptManifestFile) (needUpdate bool, err error) { + data, err := ioutil.ReadFile(path.Join(chroot, scriptObj.Path)) + if err != nil { + if !os.IsNotExist(err) { + return false, err + } + return true, nil + } else if string(data) != scriptObj.Contents.Inline { + return true, nil + } + return false, nil +} + +func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *service.Service) (needUpdate bool, err error) { + swdService, err := p.serviceManager.ReadService(serviceObj.Path) + if err != nil { + if !os.IsNotExist(err) { + return false, err + } + // service not exists + return true, nil + } else { + needChange, err := service.CompareServices(swdService, serviceObj) + if err != nil { + return false, err + } + return needChange, nil + } +} + func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool { systemService, err := p.serviceManager.ReadService(serviceObj.Path) if err != nil { @@ -330,17 +379,32 @@ func (p *K8sPlugin) servicesStateUpdate() error { return nil } -func (p *K8sPlugin) updateSwichdevService() error { - if p.updateTarget.switchdevService { - err := p.serviceManager.EnableService(p.switchdevService) +func (p *K8sPlugin) updateSwitchdevService() error { + if p.updateTarget.switchdevBeforeNMService { + err := p.serviceManager.EnableService(p.switchdevBeforeNMService) + if err != nil { + return err + } + } + + if p.updateTarget.switchdevAfterNMService { + err := p.serviceManager.EnableService(p.switchdevAfterNMService) + if err != nil { + return err + } + } + + if p.updateTarget.switchdevBeforeNMRunScript { + err := ioutil.WriteFile(path.Join(chroot, p.switchdevBeforeNMRunScript.Path), + []byte(p.switchdevBeforeNMRunScript.Contents.Inline), 0755) if err != nil { return err } } - if p.updateTarget.switchdevRunScript { - err := ioutil.WriteFile(path.Join(chroot, p.switchdevRunScript.Path), - []byte(p.switchdevRunScript.Contents.Inline), 0755) + if p.updateTarget.switchdevAfterNMRunScript { + err := ioutil.WriteFile(path.Join(chroot, p.switchdevAfterNMRunScript.Path), + []byte(p.switchdevAfterNMRunScript.Contents.Inline), 0755) if err != nil { return err }