operator: revise deployment status API and operator reconcile loop

Revised the Deployment.Status to accommodate the deployment state
conditions and driver state. Currently, Deployment has 3 conditions named
CertsVerified, CertsReady, and DriverDeployed. It also records the summary
of controller and node driver state, .i.e, no. of nodes the driver is
running.

In order to record real time status of the driver current had to rewrite
the current reconcile loop. The existing reconcile loop was keen on the
deployment CR changes and redeploy *only* the sub-objects that requires
to redeploy. Instead the new reconcile logic *refresh* all the objects
and CR status to keep the state consistent. The refresh chooses to
merge patching the objects to avoid all unnecessary updates.

There are two reconcile entry points:
- CR reconcile loop: refreshes all the sub-objects and CR status
- sub-object vent handler: redeploy only the deleted/changed resource
and updates CR status if required.

This also includes other code cleanups that come across.

TODOs:
 - E2E tests for validating if the operator restores the state of a
 broken deployment - i.e, recovering deleted/modified sub-objects.

FIXES: intel#611

pkg/apis/pmemcsi/v1alpha1/deployment_types.go

@@ -55,6 +55,7 @@ const (
 //   Related issue : https://github.com/kubernetes-sigs/controller-tools/issues/478
 // Fails setting min/max for integers: https://github.com/helm/helm/issues/5806
 
+// +k8s:deepcopy-gen=true
 // DeploymentSpec defines the desired state of Deployment
 type DeploymentSpec struct {
 	// Important: Run "make operator-generate-k8s" to regenerate code after modifying this file
@@ -109,13 +110,76 @@ type DeploymentSpec struct {
 	KubeletDir string `json:"kubeletDir,omitempty"`
 }
 
+// DeploymentConditionType type for representing a deployment status condition
+type DeploymentConditionType string
+
+const (
+	// CertsVerified means the provided deployment secrets are verified and valid for usage
+	CertsVerified DeploymentConditionType = "CertsVerified"
+	// CertsReady means secrests/certificates required for running the PMEM-CSI driver
+	// are ready and the deployment could progress further
+	CertsReady DeploymentConditionType = "CertsReady"
+	// DriverDeployed means that the all the sub-resources required for the deployment CR
+	// got created
+	DriverDeployed DeploymentConditionType = "DriverDeployed"
+)
+
+// +k8s:deepcopy-gen=true
+type DeploymentCondition struct {
+	// Type of condition.
+	Type DeploymentConditionType `json:"type"`
+	// Status of the condition, one of True, False, Unknown.
+	Status corev1.ConditionStatus `json:"status"`
+	// Message human readable text that explain why this condition is in this state
+	// +optional
+	Reason string `json:"reason,omitempty"`
+	// Last time the condition was probed.
+	// +optional
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+}
+
+type DriverType int
+
+const (
+	ControllerDriver DriverType = iota
+	NodeDriver
+)
+
+func (t DriverType) String() string {
+	switch t {
+	case ControllerDriver:
+		return "controller"
+	case NodeDriver:
+		return "node"
+	}
+	return ""
+}
+
+// +k8s:deepcopy-gen=true
+type DriverStatus struct {
+	// Type represents type of the driver: controller or node
+	Type string `json:"type"`
+	// Status represents the driver status : Ready, NotReady
+	Status string `json:"status"`
+	// Reason represents the human readable text that explains why the
+	// driver is in this state.
+	Reason string `json:"reason"`
+	// LastUpdated time of the driver status
+	LastUpdated metav1.Time `json:"lastUpdated,omitempty"`
+}
+
+// +k8s:deepcopy-gen=true
+
 // DeploymentStatus defines the observed state of Deployment
 type DeploymentStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make operator-generate-k8s" to regenerate code after modifying this file
 
 	// Phase indicates the state of the deployment
 	Phase DeploymentPhase `json:"phase,omitempty"`
+	// Conditions
+	Conditions []DeploymentCondition `json:"conditions,omitempty"`
+	Components []DriverStatus        `json:"driverComponents,omitempty"`
 	// LastUpdated time of the deployment status
 	LastUpdated metav1.Time `json:"lastUpdated,omitempty"`
 }
@@ -256,6 +320,35 @@ func (c DeploymentChange) String() string {
 	}[c]
 }
 
+func (d *Deployment) SetCondition(t DeploymentConditionType, state corev1.ConditionStatus, reason string) {
+	for _, c := range d.Status.Conditions {
+		if c.Type == t {
+			c.Status = state
+			c.Reason = reason
+			c.LastUpdateTime = metav1.Now()
+			return
+		}
+	}
+	d.Status.Conditions = append(d.Status.Conditions, DeploymentCondition{
+		Type:           t,
+		Status:         state,
+		Reason:         reason,
+		LastUpdateTime: metav1.Now(),
+	})
+}
+
+func (d *Deployment) SetDriverStatus(t DriverType, status, reason string) {
+	if d.Status.Components == nil {
+		d.Status.Components = make([]DriverStatus, 2)
+	}
+	d.Status.Components[t] = DriverStatus{
+		Type:        t.String(),
+		Status:      status,
+		Reason:      reason,
+		LastUpdated: metav1.Now(),
+	}
+}
+
 // EnsureDefaults make sure that the deployment object has all defaults set properly
 func (d *Deployment) EnsureDefaults(operatorImage string) error {
 	if d.Spec.Image == "" {
@@ -420,6 +513,30 @@ func (d *Deployment) GetOwnerReference() metav1.OwnerReference {
 	}
 }
 
+// HaveCertificatesConfigured checks if the configured deployment
+// certificate fields are valid. Returns true if valid else appropriate
+// error.
+func (d *Deployment) HaveCertificatesConfigured() (bool, error) {
+	// Encoded private keys and certificates
+	caCert := d.Spec.CACert
+	registryPrKey := d.Spec.RegistryPrivateKey
+	ncPrKey := d.Spec.NodeControllerPrivateKey
+	registryCert := d.Spec.RegistryCert
+	ncCert := d.Spec.NodeControllerCert
+
+	// sanity check
+	if caCert == nil {
+		if registryCert != nil || ncCert != nil {
+			return false, fmt.Errorf("incomplete deployment configuration: missing root CA certificate by which the provided certificates are signed")
+		}
+		return false, nil
+	} else if registryCert == nil || registryPrKey == nil || ncCert == nil || ncPrKey == nil {
+		return false, fmt.Errorf("incomplete deployment configuration: certificates and corresponding private keys must be provided")
+	}
+
+	return true, nil
+}
+
 func GetDeploymentCRDSchema() *apiextensions.JSONSchemaProps {
 	One := float64(1)
 	Hundred := float64(100)