Add status conditions and profile applied to Profile(s)

Changes:
  - report Tuned profile currently applied for each of the containerized
    Tuned daemon managed by NTO
  - report two Profile status conditions "Applied" and "Degraded"
    in every Profile indicating whether the Tuned profile was applied and
    whether there were issues during the profile application
  - cleanup of the ClusterOperator settings code; ClusterOperator now also
    reports Degraded == True if any of the Tuned Profiles failed to be
    applied cleanly for any of the containerized Tuned daemons managed by
    NTO
  - e2e test added to check the status reporting functionality
  - e2e basic/available test enhanced to check for not Degraded condition
  - using "podman build --no-cache" now.  This works around issues such as:
    containers/buildah#2837

Makefile

@@ -25,7 +25,7 @@ API_TYPES_GENERATED:=$(API_TYPES_DIR)/$(API_ZZ_GENERATED).go
 API_GO_HEADER_FILE:=pkg/apis/header.go.txt
 
 # Container image-related variables
-IMAGE_BUILD_CMD=podman build
+IMAGE_BUILD_CMD=podman build --no-cache
 IMAGE_PUSH_CMD=podman push
 DOCKERFILE=Dockerfile
 REGISTRY=quay.io

manifests/20-crd-profile.yaml

@@ -21,7 +21,7 @@ spec:
     storage: true
     schema:
       openAPIV3Schema:
-        description: Profile is a specification for a Profile resource
+        description: Profile is a specification for a Profile resource.
         type: object
         properties:
           apiVersion:
@@ -57,11 +57,48 @@ spec:
               is for internal use only and its fields may be changed/removed in the
               future.
             type: object
+            required:
+            - tunedProfile
             properties:
               bootcmdline:
                 description: kernel parameters calculated by tuned for the active
                   Tuned profile
                 type: string
+              conditions:
+                description: conditions represents the state of the per-node Profile
+                  application
+                type: array
+                items:
+                  description: ProfileStatusCondition represents a partial state of
+                    the per-node Profile application.
+                  type: object
+                  required:
+                  - lastTransitionTime
+                  - status
+                  - type
+                  properties:
+                    lastTransitionTime:
+                      description: lastTransitionTime is the time of the last update
+                        to the current status property.
+                      type: string
+                      format: date-time
+                    message:
+                      description: message provides additional information about the
+                        current condition. This is only to be consumed by humans.
+                      type: string
+                    reason:
+                      description: reason is the CamelCase reason for the condition's
+                        current status.
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: type specifies the aspect reported by this condition.
+                      type: string
               stalld:
-                description: 'deploy stall daemon: https://github.com/bristot/stalld/'
+                description: 'deploy stall daemon: https://git.kernel.org/pub/scm/utils/stalld/stalld.git'
                 type: boolean
+              tunedProfile:
+                description: the current profile in use by the Tuned daemon
+                type: string

manifests/20-crd-tuned.yaml

@@ -122,5 +122,5 @@ spec:
                         description: Name of the Tuned profile to recommend.
                         type: string
             status:
-              description: TunedStatus is the status for a Tuned resource
+              description: TunedStatus is the status for a Tuned resource.
               type: object

pkg/apis/tuned/v1/tuned_types.go

@@ -1,8 +1,10 @@
 package v1
 
 import (
-	operatorv1 "github.com/openshift/api/operator/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	operatorv1 "github.com/openshift/api/operator/v1"
 )
 
 const (
@@ -100,13 +102,13 @@ type OperandConfig struct {
 	Debug bool `json:"debug"`
 }
 
-// TunedStatus is the status for a Tuned resource
+// TunedStatus is the status for a Tuned resource.
 type TunedStatus struct {
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// TunedList is a list of Tuned resources
+// TunedList is a list of Tuned resources.
 type TunedList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
@@ -117,7 +119,7 @@ type TunedList struct {
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// Profile is a specification for a Profile resource
+// Profile is a specification for a Profile resource.
 type Profile struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -144,13 +146,68 @@ type ProfileStatus struct {
 	// kernel parameters calculated by tuned for the active Tuned profile
 	// +optional
 	Bootcmdline string `json:"bootcmdline"`
-	// deploy stall daemon: https://github.com/bristot/stalld/
+
+	// the current profile in use by the Tuned daemon
+	TunedProfile string `json:"tunedProfile"`
+
+	// conditions represents the state of the per-node Profile application
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	// +optional
+	Conditions []ProfileStatusCondition `json:"conditions,omitempty"  patchStrategy:"merge" patchMergeKey:"type"`
+
+	// deploy stall daemon: https://git.kernel.org/pub/scm/utils/stalld/stalld.git
 	// +optional
 	Stalld bool `json:"stalld"`
 }
 
+// ProfileStatusCondition represents a partial state of the per-node Profile application.
+// +k8s:deepcopy-gen=true
+type ProfileStatusCondition struct {
+	// type specifies the aspect reported by this condition.
+	// +kubebuilder:validation:Required
+	// +required
+	Type ProfileConditionType `json:"type"`
+
+	// status of the condition, one of True, False, Unknown.
+	// +kubebuilder:validation:Required
+	// +required
+	Status corev1.ConditionStatus `json:"status"`
+
+	// lastTransitionTime is the time of the last update to the current status property.
+	// +kubebuilder:validation:Required
+	// +required
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+
+	// reason is the CamelCase reason for the condition's current status.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// message provides additional information about the current condition.
+	// This is only to be consumed by humans.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// ProfileConditionType is an aspect of Tuned daemon profile application state.
+type ProfileConditionType string
+
+const (
+	// ProfileApplied indicates that the Tuned daemon has successfully applied
+	// the selected profile.
+	TunedProfileApplied ProfileConditionType = "Applied"
+
+	// TunedDegraded indicates the Tuned daemon issued errors during profile
+	// application.  To conclude the profile application was successful,
+	// both TunedProfileApplied and TunedDegraded need to be queried.
+	TunedDegraded ProfileConditionType = "Degraded"
+
+	// The number of Profile status conditions.
+	ProfileStatusConditions = 2
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// ProfileList is a list of Profile resources
+// ProfileList is a list of Profile resources.
 type ProfileList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`

pkg/operator/controller.go

@@ -37,7 +37,7 @@ import (
 	tunedinformers "github.com/openshift/cluster-node-tuning-operator/pkg/generated/informers/externalversions"
 	ntomf "github.com/openshift/cluster-node-tuning-operator/pkg/manifests"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/metrics"
-	"github.com/openshift/cluster-node-tuning-operator/pkg/tuned"
+	tunedpkg "github.com/openshift/cluster-node-tuning-operator/pkg/tuned"
 	"github.com/openshift/cluster-node-tuning-operator/pkg/util"
 	"github.com/openshift/cluster-node-tuning-operator/version"
 
@@ -268,7 +268,7 @@ func (c *Controller) sync(key wqKey) error {
 		if err != nil {
 			return fmt.Errorf("failed to sync DaemonSet: %v", err)
 		}
-		err = c.syncOperatorStatus(cr)
+		err = c.syncOperatorStatus(cr, nil)
 		if err != nil {
 			return fmt.Errorf("failed to sync OperatorStatus: %v", err)
 		}
@@ -348,7 +348,7 @@ out:
 	if err != nil {
 		lastErr = fmt.Errorf("failed to disable Pod informer: %v", err)
 	}
-	err = c.syncOperatorStatus(cr)
+	err = c.syncOperatorStatus(cr, nil)
 	if err != nil {
 		lastErr = fmt.Errorf("failed to synchronize Operator status: %v", err)
 	}
@@ -359,7 +359,7 @@ out:
 func (c *Controller) enqueueProfileUpdates() error {
 	profileList, err := c.listers.TunedProfiles.List(labels.Everything())
 	if err != nil {
-		return fmt.Errorf("failed to list Tuned profiles: %v", err)
+		return fmt.Errorf("failed to list Tuned Profiles: %v", err)
 	}
 	for _, profile := range profileList {
 		// Enqueue Profile updates into the operator's workqueue
@@ -528,6 +528,7 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error {
 			klog.V(2).Infof("syncProfile(): Profile %s not found, creating one [%s]", profileMf.Name, tunedProfileName)
 			profileMf.Spec.Config.TunedProfile = tunedProfileName
 			profileMf.Spec.Config.Debug = daemonDebug
+			profileMf.Status.Conditions = tunedpkg.InitializeStatusConditions()
 			_, err = c.clients.Tuned.TunedV1().Profiles(ntoconfig.OperatorNamespace()).Create(context.TODO(), profileMf, metav1.CreateOptions{})
 			if err != nil {
 				return fmt.Errorf("failed to create Profile %s: %v", profileMf.Name, err)
@@ -541,8 +542,15 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error {
 		return fmt.Errorf("failed to get Profile %s: %v", profileMf.Name, err)
 	}
 
+	// Profiles carry status conditions based on which OperatorStatus is also
+	// calculated.
+	err = c.syncOperatorStatus(tuned, profile)
+	if err != nil {
+		return fmt.Errorf("failed to sync OperatorStatus: %v", err)
+	}
+
 	if mcLabels != nil {
-		// The tuned profile "tunedProfileName" for nodeName matched with MachineConfig
+		// The Tuned daemon profile "tunedProfileName" for nodeName matched with MachineConfig
 		// labels set for additional machine configuration.  Sync the operator-created
 		// MachineConfig for MachineConfigPools 'pools'.
 		err := c.syncMachineConfig(getMachineConfigNameForPools(pools), mcLabels, profile.Status.Bootcmdline, profile.Status.Stalld)
@@ -597,8 +605,8 @@ func (c *Controller) syncMachineConfig(name string, labels map[string]string, bo
 		return sb.String()
 	}
 	kernelArguments = util.SplitKernelArguments(bootcmdline)
-	ignFiles = tuned.ProvideIgnitionFiles(stalld)
-	ignUnits = tuned.ProvideSystemdUnits(stalld)
+	ignFiles = tunedpkg.ProvideIgnitionFiles(stalld)
+	ignUnits = tunedpkg.ProvideSystemdUnits(stalld)
 
 	annotations := map[string]string{GeneratedByControllerVersionAnnotationKey: version.Version}
 
@@ -652,7 +660,7 @@ func (c *Controller) syncMachineConfig(name string, labels map[string]string, bo
 	return nil
 }
 
-// pruneMachineConfigs removes any MachineConfigs created by the operator that are not selected by any of the tuned profile.
+// pruneMachineConfigs removes any MachineConfigs created by the operator that are not selected by any of the Tuned daemon profile.
 func (c *Controller) pruneMachineConfigs() error {
 	mcList, err := c.listers.MachineConfigs.List(labels.Everything())
 	if err != nil {
@@ -690,7 +698,7 @@ func (c *Controller) pruneMachineConfigs() error {
 	return nil
 }
 
-// Get all operator MachineConfig names for all tuned profiles.
+// Get all operator MachineConfig names for all Tuned daemon profiles.
 func (c *Controller) getMachineConfigNamesForTuned() (map[string]bool, error) {
 	tunedList, err := c.listers.TunedResources.List(labels.Everything())
 	if err != nil {
@@ -882,7 +890,7 @@ func (c *Controller) removeResources() error {
 
 	profileList, err := c.listers.TunedProfiles.List(labels.Everything())
 	if err != nil {
-		lastErr = fmt.Errorf("failed to list Tuned profiles: %v", err)
+		lastErr = fmt.Errorf("failed to list Tuned Profiles: %v", err)
 	}
 	for _, profile := range profileList {
 		err = c.clients.Tuned.TunedV1().Profiles(ntoconfig.OperatorNamespace()).Delete(ctx, profile.Name, metav1.DeleteOptions{})