deferred: perfprof: propagate annotation

in case the performance profile is annotated with
the very same annotation which enabled tuned
deferred updates, then that annotation
is propagated to the generated tuned objects, effectively
enabling the tuned deferred update feature.

Signed-off-by: Francesco Romani <[email protected]>

pkg/performanceprofile/controller/performanceprofile/components/components.go

@@ -38,6 +38,11 @@ type Options struct {
 	ProfileMCP                  *mcov1.MachineConfigPool
 	MachineConfig               MachineConfigOptions
 	MixedCPUsFeatureGateEnabled bool
+	Tuned                       TunedOptions
+}
+
+type TunedOptions struct {
+	DeferredEnabled bool
 }
 
 type MachineConfigOptions struct {

pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go

@@ -14,6 +14,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	ntoutil "github.com/openshift/cluster-node-tuning-operator/pkg/util"
+
 	mcov1 "github.com/openshift/api/machineconfiguration/v1"
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
@@ -48,6 +50,7 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.
 	}
 	// set missing options
 	opts.MachineConfig.MixedCPUsEnabled = opts.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile)
+	opts.Tuned.DeferredEnabled = ntoutil.HasDeferredUpdateAnnotation(profile.Annotations)
 
 	ctrRuntime, err := h.getContainerRuntimeName(ctx, profile, opts.ProfileMCP)
 	if err != nil {

pkg/performanceprofile/controller/performanceprofile/components/manifestset/manifestset.go

@@ -67,7 +67,7 @@ func GetNewComponents(profile *performancev2.PerformanceProfile, opts *component
 		return nil, err
 	}
 
-	performanceTuned, err := tuned.NewNodePerformance(profile)
+	performanceTuned, err := tuned.NewNodePerformance(profile, &opts.Tuned)
 	if err != nil {
 		return nil, err
 	}

pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go

@@ -17,6 +17,7 @@ import (
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
 	profilecomponent "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
+	ntoutil "github.com/openshift/cluster-node-tuning-operator/pkg/util"
 )
 
 const (
@@ -40,8 +41,17 @@ const (
 	templatePerformanceProfileName          = "PerformanceProfileName"
 )
 
-func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.TunedRecommend) *tunedv1.Tuned {
-	return &tunedv1.Tuned{
+func OptionsFromPerformanceProfile(pp *performancev2.PerformanceProfile) components.TunedOptions {
+	opts := components.TunedOptions{}
+	if pp == nil {
+		return opts
+	}
+	opts.DeferredEnabled = ntoutil.HasDeferredUpdateAnnotation(pp.Annotations)
+	return opts
+}
+
+func newTuned(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.TunedRecommend, opts *components.TunedOptions) *tunedv1.Tuned {
+	obj := tunedv1.Tuned{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: tunedv1.SchemeGroupVersion.String(),
 			Kind:       "Tuned",
@@ -55,10 +65,14 @@ func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.Tune
 			Recommend: recommends,
 		},
 	}
+	if opts != nil {
+		obj.Annotations = ntoutil.ToggleDeferredUpdateAnnotation(obj.Annotations, opts.DeferredEnabled)
+	}
+	return &obj
 }
 
 // NewNodePerformance returns tuned profile for performance sensitive workflows
-func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tuned, error) {
+func NewNodePerformance(profile *performancev2.PerformanceProfile, opts *components.TunedOptions) (*tunedv1.Tuned, error) {
 	templateArgs := make(map[string]interface{})
 
 	templateArgs[templatePerformanceProfileName] = profile.Name
@@ -244,7 +258,7 @@ func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tun
 			MachineConfigLabels: profilecomponent.GetMachineConfigLabel(profile),
 		},
 	}
-	return new(name, profiles, recommends), nil
+	return newTuned(name, profiles, recommends, opts), nil
 }
 
 func getProfileData(tunedTemplate string, data interface{}) (string, error) {

pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go

@@ -45,15 +45,17 @@ var _ = Describe("Tuned", func() {
 	})
 
 	getTunedManifest := func(profile *performancev2.PerformanceProfile) string {
-		tuned, err := NewNodePerformance(profile)
+		opts := OptionsFromPerformanceProfile(profile)
+		tuned, err := NewNodePerformance(profile, &opts)
 		Expect(err).ToNot(HaveOccurred())
 		y, err := yaml.Marshal(tuned)
 		Expect(err).ToNot(HaveOccurred())
 		return string(y)
 	}
 
 	getTunedStructuredData := func(profile *performancev2.PerformanceProfile) *ini.File {
-		tuned, err := NewNodePerformance(profile)
+		opts := OptionsFromPerformanceProfile(profile)
+		tuned, err := NewNodePerformance(profile, &opts)
 		Expect(err).ToNot(HaveOccurred())
 		tunedData := []byte(*tuned.Spec.Profile[0].Data)
 		cfg, err := ini.Load(tunedData)
@@ -198,7 +200,8 @@ var _ = Describe("Tuned", func() {
 						HighPowerConsumption:  pointer.Bool(true),
 						PerPodPowerManagement: pointer.Bool(true),
 					}
-					_, err := NewNodePerformance(profile)
+					opts := OptionsFromPerformanceProfile(profile)
+					_, err := NewNodePerformance(profile, &opts)
 					Expect(err.Error()).To(ContainSubstring("Invalid WorkloadHints configuration: HighPowerConsumption is true and PerPodPowerManagement is true"))
 				})
 			})

pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go

@@ -24,6 +24,7 @@ import (
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift"
 	hypershiftconsts "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift/consts"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/resources"
+	ntoutil "github.com/openshift/cluster-node-tuning-operator/pkg/util"
 )
 
 var _ components.Handler = &handler{}
@@ -95,6 +96,7 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record.
 	}
 	// set missing options
 	options.MachineConfig.MixedCPUsEnabled = options.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile)
+	options.Tuned.DeferredEnabled = ntoutil.HasDeferredUpdateAnnotation(profile.Annotations)
 
 	ctrRuntime, err := h.getContainerRuntimeName(ctx, profile)
 	if err != nil {

pkg/performanceprofile/controller/performanceprofile_controller.go

@@ -596,6 +596,7 @@ func (r *PerformanceProfileReconciler) Reconcile(ctx context.Context, req ctrl.R
 			PinningMode: &pinningMode,
 		},
 		MixedCPUsFeatureGateEnabled: r.isMixedCPUsFeatureGateEnabled(),
+		Tuned:                       components.TunedOptions{}, // ensure not nil. The handler will fill later in the flow.
 	})
 	if err != nil {
 		klog.Errorf("failed to deploy performance profile %q components: %v", instance.GetName(), err)

pkg/performanceprofile/controller/performanceprofile_controller_test.go

@@ -30,6 +30,7 @@ import (
 	hcpstatus "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift/status"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/status"
 	testutils "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/utils/testing"
+	ntoutil "github.com/openshift/cluster-node-tuning-operator/pkg/util"
 	conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1"
 	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
 
@@ -271,13 +272,14 @@ var _ = Describe("Controller", func() {
 
 			It("should remove outdated tuned objects", func() {
 				skipForHypershift()
-				tunedOutdatedA, err := tuned.NewNodePerformance(profile)
+				opts := tuned.OptionsFromPerformanceProfile(profile)
+				tunedOutdatedA, err := tuned.NewNodePerformance(profile, &opts)
 				Expect(err).ToNot(HaveOccurred())
 				tunedOutdatedA.Name = "outdated-a"
 				tunedOutdatedA.OwnerReferences = []metav1.OwnerReference{
 					{Name: profile.Name},
 				}
-				tunedOutdatedB, err := tuned.NewNodePerformance(profile)
+				tunedOutdatedB, err := tuned.NewNodePerformance(profile, &opts)
 				Expect(err).ToNot(HaveOccurred())
 				tunedOutdatedB.Name = "outdated-b"
 				tunedOutdatedB.OwnerReferences = []metav1.OwnerReference{
@@ -373,7 +375,8 @@ var _ = Describe("Controller", func() {
 					kc, err = kubeletconfig.New(profile, &components.KubeletConfigOptions{MachineConfigPoolSelector: map[string]string{mcpSelectorKey: mcpSelectorValue}})
 					Expect(err).ToNot(HaveOccurred())
 
-					tunedPerformance, err = tuned.NewNodePerformance(profile)
+					opts := tuned.OptionsFromPerformanceProfile(profile)
+					tunedPerformance, err = tuned.NewNodePerformance(profile, &opts)
 					Expect(err).ToNot(HaveOccurred())
 
 					runtimeClass = runtimeclass.New(profile, machineconfig.HighPerformanceRuntime)
@@ -837,6 +840,51 @@ var _ = Describe("Controller", func() {
 			})
 		})
 
+		Context("with the deferred annotation set", func() {
+			It("should propagate the deferred annotation only in the tuned object", func() {
+				prof := profile.DeepCopy()
+				prof.Annotations = ntoutil.ToggleDeferredUpdateAnnotation(prof.Annotations, true)
+				instance = prof
+				if isHypershift {
+					instance = encapsulateProfileInConfigMap(prof)
+				}
+				r := newFakeReconciler(instance, profileMCP, infra, clusterOperator)
+
+				Expect(reconcileTimes(r, request, 2)).To(Equal(reconcile.Result{}))
+
+				mcKey := types.NamespacedName{
+					Name:      machineconfig.GetMachineConfigName(profile.Name),
+					Namespace: metav1.NamespaceNone,
+				}
+
+				mc := &mcov1.MachineConfig{}
+				Expect(r.ManagementClient.Get(context.TODO(), mcKey, mc)).To(Succeed())
+				Expect(ntoutil.HasDeferredUpdateAnnotation(mc.Annotations)).To(BeFalse())
+
+				genericKey := types.NamespacedName{
+					Name:      components.GetComponentName(prof.Name, components.ComponentNamePrefix),
+					Namespace: metav1.NamespaceNone,
+				}
+
+				kc := &mcov1.KubeletConfig{}
+				Expect(r.ManagementClient.Get(context.TODO(), genericKey, kc)).To(Succeed())
+				Expect(ntoutil.HasDeferredUpdateAnnotation(kc.Annotations)).To(BeFalse())
+
+				runtimeClass := &nodev1.RuntimeClass{}
+				Expect(r.Get(context.TODO(), genericKey, runtimeClass)).To(Succeed())
+				Expect(ntoutil.HasDeferredUpdateAnnotation(runtimeClass.Annotations)).To(BeFalse())
+
+				// the tuned object must be the only one to have the annotation propagated
+				tunedPerformance := &tunedv1.Tuned{}
+				tunedKey := types.NamespacedName{
+					Name:      components.GetComponentName(prof.Name, components.ProfileNamePerformance),
+					Namespace: components.NamespaceNodeTuningOperator,
+				}
+				Expect(r.ManagementClient.Get(context.TODO(), tunedKey, tunedPerformance)).To(Succeed())
+				Expect(ntoutil.HasDeferredUpdateAnnotation(tunedPerformance.Annotations)).To(BeTrue())
+			})
+		})
+
 		Context("with profile with deletion timestamp", func() {
 			BeforeEach(func() {
 				skipForHypershift()
@@ -854,7 +902,8 @@ var _ = Describe("Controller", func() {
 				kc, err := kubeletconfig.New(profile, &components.KubeletConfigOptions{MachineConfigPoolSelector: map[string]string{mcpSelectorKey: mcpSelectorValue}})
 				Expect(err).ToNot(HaveOccurred())
 
-				tunedPerformance, err := tuned.NewNodePerformance(profile)
+				opts := tuned.OptionsFromPerformanceProfile(profile)
+				tunedPerformance, err := tuned.NewNodePerformance(profile, &opts)
 				Expect(err).ToNot(HaveOccurred())
 
 				runtimeClass := runtimeclass.New(profile, machineconfig.HighPerformanceRuntime)

pkg/tuned/cmd/render/render.go

@@ -173,7 +173,8 @@ func render(inputDir []string, outputDir string, mcpName string) error {
 		klog.Infof("No PerformanceProfile found for MachineConfigPool %s", mcpName)
 	}
 	for _, pp := range filteredPerformanceProfiles {
-		tunedFromPP, err := tuned.NewNodePerformance(pp)
+		tunedOpts := tuned.OptionsFromPerformanceProfile(pp)
+		tunedFromPP, err := tuned.NewNodePerformance(pp, &tunedOpts)
 		if err != nil {
 			e := fmt.Errorf("unable to get tuned from PerformanceProfile:%s. error: %w", pp.Name, err)
 			klog.Error(e)

test/e2e/performanceprofile/functests/8_performance_workloadhints/workloadhints.go

@@ -915,7 +915,8 @@ func getNodeNames(nodes []corev1.Node) []string {
 }
 
 func getTunedStructuredData(profile *performancev2.PerformanceProfile) *ini.File {
-	tuned, err := tuned.NewNodePerformance(profile)
+	opts := tuned.OptionsFromPerformanceProfile(profile)
+	tuned, err := tuned.NewNodePerformance(profile, &opts)
 	Expect(err).ToNot(HaveOccurred())
 	tunedData := []byte(*tuned.Spec.Profile[0].Data)
 	cfg, err := ini.Load(tunedData)