diff --git a/test/e2e/performanceprofile/functests/2_performance_update/tuned_deferred.go b/test/e2e/performanceprofile/functests/2_performance_update/tuned_deferred.go new file mode 100644 index 000000000..c36787320 --- /dev/null +++ b/test/e2e/performanceprofile/functests/2_performance_update/tuned_deferred.go @@ -0,0 +1,374 @@ +package __performance_update + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "gopkg.in/ini.v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" + + testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" + testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" + hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" + testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/mcps" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodepools" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate" +) + +var _ = Describe("Tuned Deferred tests of performance profile", Ordered, func() { + var ( + workerCNFNodes []corev1.Node + err error + poolName string + np *hypershiftv1beta1.NodePool + tuned *tunedv1.Tuned + ) + + type TunedProfileConfig struct { + DeferMode string + ProfileChangeType string // "first-time" or "in-place" + ExpectedBehavior string // "deferred" or "immediate" + Kernel_shmmni string // 4096 or 8192 + } + + name := "performance-patch" + + BeforeEach(func() { + if discovery.Enabled() && testutils.ProfileNotFound { + Skip("Discovery mode enabled, performance profile not found") + } + workerCNFNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + workerCNFNodes, err = nodes.MatchingOptionalSelector(workerCNFNodes) + Expect(err).ToNot(HaveOccurred(), "error looking for the optional selector: %v", err) + Expect(workerCNFNodes).ToNot(BeEmpty()) + + profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + + if hypershift.IsHypershiftCluster() { + hostedClusterName, err := hypershift.GetHostedClusterName() + Expect(err).ToNot(HaveOccurred()) + np, err = nodepools.GetByClusterName(context.TODO(), testclient.ControlPlaneClient, hostedClusterName) + Expect(err).ToNot(HaveOccurred()) + poolName = client.ObjectKeyFromObject(np).String() + testlog.Infof("using NodePool: %q", poolName) + err = nodepools.WaitForConfigToBeReady(context.TODO(), testclient.ControlPlaneClient, np.Name, np.Namespace) + Expect(err).ToNot(HaveOccurred()) + } else { + poolName, err = mcps.GetByProfile(profile) + Expect(err).ToNot(HaveOccurred()) + testlog.Infof("using performanceMCP: %q", poolName) + } + + }) + + Context("Tuned Deferred status", func() { + DescribeTable("Validate Tuned DeferMode behavior", + func(tc TunedProfileConfig) { + + if tc.ProfileChangeType == "first-time" { + tuned = getTunedProfile(name) + + // If profile is present + if tuned != nil { + + // Delete the profile + testlog.Infof("Deleting Tuned Profile Patch: %s", name) + Expect(testclient.ControlPlaneClient.Delete(context.TODO(), tuned)).To(Succeed()) + time.Sleep(2 * time.Minute) + if hypershiftutils.IsHypershiftCluster() { + // This will trigger a reboot + By("De-attaching the tuning object to the second node pool") + Expect(nodepools.DeattachTuningObject(context.TODO(), testclient.DataPlaneClient, tuned)).To(Succeed()) + By("Waiting for the nodepool configuration to start updating") + err := nodepools.WaitForUpdatingConfig(context.TODO(), testclient.ControlPlaneClient, np.Name, np.Namespace) + Expect(err).ToNot(HaveOccurred()) + By("Waiting for the nodepool configuration to be ready") + err = nodepools.WaitForConfigToBeReady(context.TODO(), testclient.ControlPlaneClient, np.Name, np.Namespace) + Expect(err).ToNot(HaveOccurred()) + } + } + tuned, err = createTunedObject(tc.DeferMode, tc.Kernel_shmmni, np, name) + Expect(err).NotTo(HaveOccurred()) + + } else { + tuned := getTunedProfile(name) // Get the tuned profile + tunedData := []byte(*tuned.Spec.Profile[0].Data) + + cfg, err := ini.Load(tunedData) + Expect(err).ToNot(HaveOccurred()) + + sysctlSection, err := cfg.GetSection("sysctl") + Expect(err).ToNot(HaveOccurred()) + + sysctlSection.Key("kernel.shmmni").SetValue(tc.Kernel_shmmni) + Expect(sysctlSection.Key("kernel.shmmni").String()).To(Equal(tc.Kernel_shmmni)) + + var updatedTunedData bytes.Buffer + _, err = cfg.WriteTo(&updatedTunedData) + Expect(err).ToNot(HaveOccurred()) + + updatedTunedDataBytes := updatedTunedData.Bytes() + tuned.Spec.Profile[0].Data = stringPtr(string(updatedTunedDataBytes)) // Update the pointer to the new data + + testlog.Infof("Updating Tuned Profile Patch: %s", name) + + // Update the tuned profile with the new data + Expect(testclient.ControlPlaneClient.Update(context.TODO(), tuned)).To(Succeed()) + + // Fetch the updated profile and verify the change + tuned = getTunedProfile(name) + tunedData = []byte(*tuned.Spec.Profile[0].Data) + } + + switch tc.ExpectedBehavior { + case "immediate": + verifyImmediateApplication(tuned, workerCNFNodes, tc.Kernel_shmmni) + case "deferred": + out, err := verifyDeferredApplication(tuned, workerCNFNodes, tc.Kernel_shmmni, name) + Expect(err).To(BeNil()) + Expect(out).To(BeTrue()) + rebootNode(poolName) + verifyImmediateApplication(tuned, workerCNFNodes, tc.Kernel_shmmni) + } + }, + Entry("DeferAlways with first-time profile change", TunedProfileConfig{ + DeferMode: "always", + ProfileChangeType: "first-time", + ExpectedBehavior: "deferred", + Kernel_shmmni: "8192", + }), + Entry("DeferAlways with in-place profile update", TunedProfileConfig{ + DeferMode: "always", + ProfileChangeType: "in-place", + ExpectedBehavior: "deferred", + Kernel_shmmni: "4096", + }), + Entry("DeferUpdate with first-time profile change", TunedProfileConfig{ + DeferMode: "update", + ProfileChangeType: "first-time", + ExpectedBehavior: "immediate", + Kernel_shmmni: "8192", + }), + Entry("DeferUpdate with in-place profile update", TunedProfileConfig{ + DeferMode: "update", + ProfileChangeType: "in-place", + ExpectedBehavior: "deferred", + Kernel_shmmni: "4096", + }), + Entry("No annotation (default behavior) with first-time profile change", TunedProfileConfig{ + DeferMode: "", // No defer annotation + ProfileChangeType: "first-time", + ExpectedBehavior: "immediate", + Kernel_shmmni: "8192", + }), + Entry("Never mode with in-place profile update", TunedProfileConfig{ + DeferMode: "never", + ProfileChangeType: "in-place", + ExpectedBehavior: "immediate", + Kernel_shmmni: "4096", + }), + ) + }) + + AfterAll(func() { + tuned = getTunedProfile(name) + if tuned != nil { + Expect(testclient.ControlPlaneClient.Delete(context.TODO(), tuned)).To(Succeed()) + } + }) + +}) + +func stringPtr(s string) *string { + return &s +} + +func getTunedProfile(tunedName string) *tunedv1.Tuned { + tunedList := &tunedv1.TunedList{} + //tunedName := "performance-patch" // Replace this with your dynamic name if needed + var matchedTuned *tunedv1.Tuned + + Eventually(func(g Gomega) { + // List all Tuned objects in the namespace + g.Expect(testclient.DataPlaneClient.List(context.TODO(), tunedList, &client.ListOptions{ + Namespace: components.NamespaceNodeTuningOperator, + })).To(Succeed()) + + // Ensure there are Tuned items to process + g.Expect(len(tunedList.Items)).To(BeNumerically(">", 1)) + + // Find the Tuned object with the name "performance-patch" + for _, tuned := range tunedList.Items { + if tuned.Name == tunedName { + matchedTuned = &tuned + break + } + } + + }).WithTimeout(time.Minute * 3).WithPolling(time.Second * 10).Should(Succeed()) + + // Check if no matching Tuned profile was found + if matchedTuned == nil { + return nil + } + // Return the matched Tuned object + return matchedTuned +} + +// CreateTuned creates a Tuned profile. +func createTunedObject(deferMode string, kernel_shmmni string, np *hypershiftv1beta1.NodePool, name string) (*tunedv1.Tuned, error) { + tunedName := name + ns := components.NamespaceNodeTuningOperator + priority := uint64(19) + data := fmt.Sprintf(` + [main] + summary=Configuration changes profile inherited from performance created tuned + include=openshift-node-performance-performance + [sysctl] + kernel.shmmni=%s + `, kernel_shmmni) + + // Create a Tuned object + tuned := &tunedv1.Tuned{ + TypeMeta: metav1.TypeMeta{ + APIVersion: tunedv1.SchemeGroupVersion.String(), + Kind: "Tuned", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: tunedName, + Namespace: ns, + Annotations: map[string]string{ + "tuned.openshift.io/deferred": deferMode, + }, + }, + Spec: tunedv1.TunedSpec{ + Profile: []tunedv1.TunedProfile{ + { + Name: &tunedName, + Data: &data, + }, + }, + Recommend: []tunedv1.TunedRecommend{ + { + MachineConfigLabels: map[string]string{"machineconfiguration.openshift.io/role": testutils.RoleWorkerCNF}, + Priority: &priority, + Profile: &tunedName, + }, + }, + }, + } + + // Create the Tuned object in the cluster + testlog.Infof("Creating Tuned Profile Patch: %s", name) + Expect(testclient.ControlPlaneClient.Create(context.TODO(), tuned)).To(Succeed()) + if hypershiftutils.IsHypershiftCluster() { + By("Attaching the tuning object to the second node pool") + Expect(nodepools.AttachTuningObject(context.TODO(), testclient.ControlPlaneClient, tuned)).To(Succeed()) + + By("Waiting for the nodepool configuration to start updating") + err := nodepools.WaitForUpdatingConfig(context.TODO(), testclient.ControlPlaneClient, np.Name, np.Namespace) + Expect(err).ToNot(HaveOccurred()) + + By("Waiting for the nodepool configuration to be ready") + err = nodepools.WaitForConfigToBeReady(context.TODO(), testclient.ControlPlaneClient, np.Name, np.Namespace) + Expect(err).ToNot(HaveOccurred()) + } + return tuned, nil +} + +func execSysctlOnWorkers(ctx context.Context, workerNodes []corev1.Node, sysctlMap map[string]string) { + var err error + var out []byte + isSNO := false + for _, node := range workerNodes { + for param, expected := range sysctlMap { + By(fmt.Sprintf("executing the command \"sysctl -n %s\"", param)) + tunedCmd := []string{"/bin/sh", "-c", "chroot /host sysctl -a | grep shmmni"} + tunedPod := nodes.TunedForNode(&node, isSNO) + out, err = pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) + Expect(err).ToNot(HaveOccurred()) + Expect(strings.TrimSpace(strings.SplitN(string(out), "=", 2)[1])).Should(Equal(expected), "parameter %s value is not %s.", out, expected) + } + } +} + +func verifyImmediateApplication(tuned *tunedv1.Tuned, workerCNFNodes []corev1.Node, kernel_shmmni string) { + time.Sleep(40 * time.Second) + sysctlMap := map[string]string{ + "kernel.shmmni": kernel_shmmni, + } + execSysctlOnWorkers(context.TODO(), workerCNFNodes, sysctlMap) +} + +func verifyDeferredApplication(tuned *tunedv1.Tuned, workerCNFNodes []corev1.Node, kernel_shmmni string, name string) (bool, error) { + time.Sleep(20 * time.Second) + expectedMessage := fmt.Sprintf("The TuneD daemon profile is waiting for the next node restart: %s", name) + profile := &tunedv1.Profile{} + node := workerCNFNodes[0] + err := testclient.DataPlaneClient.Get(context.TODO(), client.ObjectKey{Name: node.Name, Namespace: tuned.Namespace}, profile) + if err != nil { + return false, fmt.Errorf("failed to get profile: %w", err) + } + + // Iterate over the conditions to find the matching message + for _, condition := range profile.Status.Conditions { + if condition.Message == expectedMessage { + return true, nil + } + } + + // Return false if the condition message is not found + return false, nil +} + +func rebootNode(poolName string) { + profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + if err != nil { + testlog.Errorf("Unable to fetch latest performance profile err: %v", err) + } + testlog.Info("Rebooting the node") + // reboot the node, for that we change the numa policy to best-effort + // Note: this is used only to trigger reboot + policy := "best-effort" + // Need to make some changes to pp , causing system reboot + currentPolicy := profile.Spec.NUMA.TopologyPolicy + if *currentPolicy == "best-effort" { + policy = "single-numa-node" + } + profile.Spec.NUMA = &performancev2.NUMA{ + TopologyPolicy: &policy, + } + + By("Updating the performance profile") + profiles.UpdateWithRetry(profile) + + By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName)) + profilesupdate.WaitForTuningUpdating(context.TODO(), profile) + + By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) + profilesupdate.WaitForTuningUpdated(context.TODO(), profile) + +}