Merge pull request #428 from vyzigold/autoscaling_custom_TLS

[OSPRH-8065] Rework custom prometheus CA
openstack-k8s-operators · Jul 5, 2024 · 3c3fb82 · 3c3fb82
2 parents 8847cf7 + 48c7c78
commit 3c3fb82
Show file tree

Hide file tree

Showing 10 changed files with 133 additions and 25 deletions.
diff --git a/api/bases/telemetry.openstack.org_autoscalings.yaml b/api/bases/telemetry.openstack.org_autoscalings.yaml
@@ -361,9 +361,26 @@ spec:
                 maximum: 65535
                 minimum: 1
                 type: integer
-              prometheusTLS:
-                description: If TLS should be used for user deployed prometheus
-                type: boolean
+              prometheusTLSCaCertSecret:
+                description: If defined, specifies which CA certificate to use for
+                  user deployed prometheus
+                nullable: true
+                properties:
+                  key:
+                    description: The key of the secret to select from.  Must be a
+                      valid secret key.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                required:
+                - key
+                type: object
+                x-kubernetes-map-type: atomic
             required:
             - heatInstance
             type: object

diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml
@@ -379,9 +379,27 @@ spec:
                     maximum: 65535
                     minimum: 1
                     type: integer
-                  prometheusTLS:
-                    description: If TLS should be used for user deployed prometheus
-                    type: boolean
+                  prometheusTLSCaCertSecret:
+                    description: If defined, specifies which CA certificate to use
+                      for user deployed prometheus
+                    nullable: true
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be
+                          a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be
+                          defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                    x-kubernetes-map-type: atomic
                 required:
                 - heatInstance
                 type: object

diff --git a/api/v1beta1/autoscaling_types.go b/api/v1beta1/autoscaling_types.go
@@ -20,6 +20,7 @@ import (
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
@@ -155,9 +156,10 @@ type AutoscalingSpecBase struct {
 	// +kubebuilder:validation:Optional
 	PrometheusPort int32 `json:"prometheusPort,omitempty"`
 
-	// If TLS should be used for user deployed prometheus
+	// If defined, specifies which CA certificate to use for user deployed prometheus
 	// +kubebuilder:validation:Optional
-	PrometheusTLS *bool `json:"prometheusTLS,omitempty"`
+	// +nullable
+	PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"`
 
 	// Heat instance name.
 	// +kubebuilder:default=heat

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml
@@ -361,9 +361,26 @@ spec:
                 maximum: 65535
                 minimum: 1
                 type: integer
-              prometheusTLS:
-                description: If TLS should be used for user deployed prometheus
-                type: boolean
+              prometheusTLSCaCertSecret:
+                description: If defined, specifies which CA certificate to use for
+                  user deployed prometheus
+                nullable: true
+                properties:
+                  key:
+                    description: The key of the secret to select from.  Must be a
+                      valid secret key.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                required:
+                - key
+                type: object
+                x-kubernetes-map-type: atomic
             required:
             - heatInstance
             type: object

diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml
@@ -379,9 +379,27 @@ spec:
                     maximum: 65535
                     minimum: 1
                     type: integer
-                  prometheusTLS:
-                    description: If TLS should be used for user deployed prometheus
-                    type: boolean
+                  prometheusTLSCaCertSecret:
+                    description: If defined, specifies which CA certificate to use
+                      for user deployed prometheus
+                    nullable: true
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be
+                          a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be
+                          defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                    x-kubernetes-map-type: atomic
                 required:
                 - heatInstance
                 type: object

diff --git a/controllers/autoscaling_controller.go b/controllers/autoscaling_controller.go
@@ -393,7 +393,7 @@ func (r *AutoscalingReconciler) reconcileNormal(
 	} else {
 		instance.Status.PrometheusPort = instance.Spec.PrometheusPort
 	}
-	if instance.Spec.PrometheusTLS == nil {
+	if instance.Spec.PrometheusTLSCaCertSecret == nil {
 		metricStorage := &telemetryv1.MetricStorage{}
 		err := r.Client.Get(ctx, client.ObjectKey{
 			Namespace: instance.Namespace,
@@ -409,7 +409,7 @@ func (r *AutoscalingReconciler) reconcileNormal(
 		}
 		instance.Status.PrometheusTLS = metricStorage.Spec.PrometheusTLS.Enabled()
 	} else {
-		instance.Status.PrometheusTLS = *instance.Spec.PrometheusTLS
+		instance.Status.PrometheusTLS = true
 	}
 
 	db, result, err := r.ensureDB(ctx, helper, instance)
@@ -560,10 +560,14 @@ func (r *AutoscalingReconciler) generateServiceConfig(
 	}
 
 	prometheusParams := map[string]interface{}{
-		"Host":   instance.Status.PrometheusHost,
-		"Port":   instance.Status.PrometheusPort,
-		"TLS":    instance.Status.PrometheusPort,
-		"CaCert": tls.DownstreamTLSCABundlePath,
+		"Host": instance.Status.PrometheusHost,
+		"Port": instance.Status.PrometheusPort,
+		"TLS":  instance.Status.PrometheusTLS,
+	}
+	if instance.Spec.PrometheusTLSCaCertSecret != nil {
+		prometheusParams["CaCert"] = autoscaling.CustomPrometheusCaCertFolderPath + instance.Spec.PrometheusTLSCaCertSecret.Key
+	} else {
+		prometheusParams["CaCert"] = tls.DownstreamTLSCABundlePath
 	}
 	templateParameters["Prometheus"] = prometheusParams
 

diff --git a/pkg/autoscaling/aodh_statefulset.go b/pkg/autoscaling/aodh_statefulset.go
@@ -83,7 +83,7 @@ func AodhStatefulSet(
 	notifierVolumeMounts := getVolumeMounts("aodh-notifier")
 	listenerVolumeMounts := getVolumeMounts("aodh-listener")
 
-	// add CA cert if defined
+	// add openstack CA cert if defined
 	if instance.Spec.Aodh.TLS.CaBundleSecretName != "" {
 		volumes = append(volumes, instance.Spec.Aodh.TLS.CreateVolume())
 		apiVolumeMounts = append(apiVolumeMounts, instance.Spec.Aodh.TLS.CreateVolumeMounts(nil)...)
@@ -92,6 +92,12 @@ func AodhStatefulSet(
 		listenerVolumeMounts = append(listenerVolumeMounts, instance.Spec.Aodh.TLS.CreateVolumeMounts(nil)...)
 	}
 
+	// add prometheus CA cert if defined
+	if instance.Spec.PrometheusTLSCaCertSecret != nil {
+		volumes = append(volumes, getCustomPrometheusCaVolume(instance.Spec.PrometheusTLSCaCertSecret.LocalObjectReference.Name))
+		evaluatorVolumeMounts = append(evaluatorVolumeMounts, getCustomPrometheusCaVolumeMount(instance.Spec.PrometheusTLSCaCertSecret.Key))
+	}
+
 	for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} {
 		if instance.Spec.Aodh.TLS.API.Enabled(endpt) {
 			var tlsEndptCfg tls.GenericService

diff --git a/pkg/autoscaling/const.go b/pkg/autoscaling/const.go
@@ -38,6 +38,9 @@ const (
 
 	// AodhAPIPort -
 	AodhAPIPort = 8042
+
+	// CustomPrometheusCaCertFolderPath -
+	CustomPrometheusCaCertFolderPath = "/etc/pki/ca-trust/extracted/pem/"
 )
 
 // PrometheusReplicas -

diff --git a/pkg/autoscaling/volumes.go b/pkg/autoscaling/volumes.go
@@ -76,3 +76,25 @@ func getVolumeMounts(serviceName string) []corev1.VolumeMount {
 		},
 	}
 }
+
+// getCustomPrometheusCaVolume - Volume for CA certificate of user deployed Prometheus
+func getCustomPrometheusCaVolume(secretName string) corev1.Volume {
+	return corev1.Volume{
+		Name: "custom-prometheus-ca",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: secretName,
+			},
+		},
+	}
+}
+
+// getCustomPrometheusCaVolumeMount - VolumeMount for CA certificate of user deployed Prometheus
+func getCustomPrometheusCaVolumeMount(fileName string) corev1.VolumeMount {
+	return corev1.VolumeMount{
+		Name:      "custom-prometheus-ca",
+		MountPath: CustomPrometheusCaCertFolderPath + fileName,
+		SubPath:   fileName,
+		ReadOnly:  true,
+	}
+}