openshift, operator: Use cert service at openshift (nmstate#1263)

This change add annotations to use openshift cert service and will detect
openshift clusters and do the following:
- Skip nmstate cert-manager installation
- Remove obsolete nmstate-cert-manager deployment

Signed-off-by: Enrique Llorente <[email protected]>

controllers/operator/nmstate_controller.go

@@ -33,8 +33,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/openshift/cluster-network-operator/pkg/apply"
@@ -116,6 +118,10 @@ func (r *NMStateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, err
 	}
 
+	if err := r.cleanupObsoleteResources(ctx, instance.Namespace); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	r.Log.Info("Reconcile complete.")
 	return ctrl.Result{}, nil
 }
@@ -311,6 +317,12 @@ func (r *NMStateReconciler) applyHandler(instance *nmstatev1.NMState) error {
 	data.Data["HandlerAffinity"] = handlerAffinity
 	data.Data["SelfSignConfiguration"] = selfSignConfiguration
 
+	isOpenShift, err := cluster.IsOpenShift(r.APIClient)
+	if err != nil {
+		return err
+	}
+	data.Data["IsOpenShift"] = isOpenShift
+
 	return r.renderAndApply(instance, data, "handler", true)
 }
 
@@ -346,6 +358,26 @@ func (r *NMStateReconciler) patchOpenshiftConsolePlugin(ctx context.Context) err
 	return nil
 }
 
+func (r *NMStateReconciler) cleanupObsoleteResources(ctx context.Context, namespace string) error {
+	isOpenShift, err := cluster.IsOpenShift(r.APIClient)
+	if err != nil {
+		return err
+	}
+	// We are no longer using cert-manager at openshift, let's remove it
+	if isOpenShift {
+		err = r.Client.Delete(ctx, &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      os.Getenv("HANDLER_PREFIX") + "nmstate-cert-manager",
+			},
+		})
+		if !apierrors.IsNotFound(err) {
+			return fmt.Errorf("failed deleting obsolete cert-manager deployment at openshift: %w", err)
+		}
+	}
+	return nil
+}
+
 // webhookReplicaCount returns the number of replicas for the nmstate webhook
 // deployment based on the underlying infrastructure topology. It returns 2
 // values (and error):

deploy/handler/operator.yaml

@@ -182,6 +182,7 @@ spec:
         - name: tls-key-pair
           secret:
             secretName: {{template "handlerPrefix" .}}nmstate-webhook
+{{- if not .IsOpenShift }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -268,6 +269,7 @@ spec:
               value: {{ .SelfSignConfiguration.CertRotateInterval }}
             - name: CERT_OVERLAP_INTERVAL
               value: {{ .SelfSignConfiguration.CertOverlapInterval }}
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -388,6 +390,8 @@ kind: Service
 metadata:
   name: {{template "handlerPrefix" .}}nmstate-webhook
   namespace: {{ .HandlerNamespace }}
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: {{template "handlerPrefix" .}}nmstate-webhook
   labels:
     app: kubernetes-nmstate
 spec:
@@ -420,6 +424,8 @@ apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   name: {{template "handlerPrefix" .}}nmstate
+  annotations:
+    service.beta.openshift.io/inject-cabundle: "true"
   labels:
     app: kubernetes-nmstate
 webhooks:

test/e2e/operator/operator.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	nmstatev1 "github.com/nmstate/kubernetes-nmstate/api/v1"
+	"github.com/nmstate/kubernetes-nmstate/pkg/cluster"
 	"github.com/nmstate/kubernetes-nmstate/test/cmd"
 	"github.com/nmstate/kubernetes-nmstate/test/e2e/daemonset"
 	"github.com/nmstate/kubernetes-nmstate/test/e2e/deployment"
@@ -119,8 +120,10 @@ func EventuallyOperandIsReady(testData TestData) {
 	daemonset.GetEventually(testData.HandlerKey).Should(daemonset.BeReady(), "should start handler daemonset")
 	By("Wait deployment webhook is ready")
 	deployment.GetEventually(testData.WebhookKey).Should(deployment.BeReady(), "should start webhook deployment")
-	By("Wait deployment cert-manager is ready")
-	deployment.GetEventually(testData.CertManagerKey).Should(deployment.BeReady(), "should start cert-manager deployment")
+	if !IsOpenShift() {
+		By("Wait deployment cert-manager is ready")
+		deployment.GetEventually(testData.CertManagerKey).Should(deployment.BeReady(), "should start cert-manager deployment")
+	}
 	if testData.MetricsKey != nil {
 		By("Wait deployment metrics is ready")
 		deployment.GetEventually(*testData.MetricsKey).Should(deployment.BeReady(), "should start metrics deployment")
@@ -130,7 +133,9 @@ func EventuallyOperandIsReady(testData TestData) {
 func EventuallyOperandIsNotFound(testData TestData) {
 	EventuallyIsNotFound(testData.HandlerKey, &appsv1.DaemonSet{}, "should delete handler daemonset")
 	EventuallyIsNotFound(testData.WebhookKey, &appsv1.Deployment{}, "should delete webhook deployment")
-	EventuallyIsNotFound(testData.CertManagerKey, &appsv1.Deployment{}, "should delete cert-manager deployment")
+	if !IsOpenShift() {
+		EventuallyIsNotFound(testData.CertManagerKey, &appsv1.Deployment{}, "should delete cert-manager deployment")
+	}
 	if testData.MetricsKey != nil {
 		EventuallyIsNotFound(*testData.MetricsKey, &appsv1.Deployment{}, "should delete metrics deployment")
 	}
@@ -149,7 +154,9 @@ func EventuallyOperandIsNotFound(testData TestData) {
 func EventuallyOperandIsFound(testData TestData) {
 	EventuallyIsFound(testData.HandlerKey, &appsv1.DaemonSet{}, "should create handler daemonset")
 	EventuallyIsFound(testData.WebhookKey, &appsv1.Deployment{}, "should create webhook deployment")
-	EventuallyIsFound(testData.CertManagerKey, &appsv1.Deployment{}, "should create cert-manager deployment")
+	if !IsOpenShift() {
+		EventuallyIsFound(testData.CertManagerKey, &appsv1.Deployment{}, "should create cert-manager deployment")
+	}
 	if testData.MetricsKey != nil {
 		EventuallyIsFound(*testData.MetricsKey, &appsv1.Deployment{}, "should create metrics deployment")
 	}
@@ -184,3 +191,10 @@ func UninstallOperator(operator TestData) {
 	Expect(testenv.Client.Delete(context.TODO(), &ns)).To(SatisfyAny(Succeed(), WithTransform(apierrors.IsNotFound, BeTrue())))
 	EventuallyIsNotFound(types.NamespacedName{Name: operator.Ns}, &ns, "should delete the namespace")
 }
+
+func IsOpenShift() bool {
+	GinkgoHelper()
+	isOpenShift, err := cluster.IsOpenShift(testenv.Client)
+	Expect(err).ToNot(HaveOccurred())
+	return isOpenShift
+}