Merge pull request #518 from securesign/fix-service-monitors

SECURESIGN-1250 | Targets are not coming up with monitoring enabled.

bundle/manifests/rhtas.redhat.com_rekors.yaml

@@ -150,6 +150,9 @@ spec:
                     x-kubernetes-validations:
                     - message: Feature cannot be disabled
                       rule: (self || !oldSelf)
+                  host:
+                    description: Set hostname for your Ingress/Route.
+                    type: string
                 required:
                 - enabled
                 type: object

bundle/manifests/rhtas.redhat.com_securesigns.yaml

@@ -512,6 +512,9 @@ spec:
                         x-kubernetes-validations:
                         - message: Feature cannot be disabled
                           rule: (self || !oldSelf)
+                      host:
+                        description: Set hostname for your Ingress/Route.
+                        type: string
                     required:
                     - enabled
                     type: object

internal/clidownload/component.go

@@ -58,7 +58,7 @@ func (c *Component) Start(ctx context.Context) error {
 
 	obj = append(obj, ns)
 	obj = append(obj, c.createDeployment(ns.Name, labels))
-	svc := kubernetes.CreateService(ns.Name, cliServerName, cliServerPortName, cliServerPort, labels)
+	svc := kubernetes.CreateService(ns.Name, cliServerName, cliServerPortName, cliServerPort, cliServerPort, labels)
 	obj = append(obj, svc)
 	ingress, err := kubernetes.CreateIngress(ctx, c.Client, *svc, rhtasv1alpha1.ExternalAccess{Host: CliHostName}, cliServerPortName, labels)
 	if err != nil {

internal/controller/common/utils/kubernetes/service.go

@@ -12,7 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func CreateService(namespace string, name string, portName string, port int, labels map[string]string) *corev1.Service {
+func CreateService(namespace string, name string, portName string, port int, targetPort int32, labels map[string]string) *corev1.Service {
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -26,7 +26,7 @@ func CreateService(namespace string, name string, portName string, port int, lab
 					Name:       portName,
 					Protocol:   corev1.ProtocolTCP,
 					Port:       int32(port),
-					TargetPort: intstr.FromInt(port),
+					TargetPort: intstr.FromInt32(targetPort),
 				},
 			},
 		},

internal/controller/ctlog/actions/constants.go

@@ -6,7 +6,10 @@ const (
 	RBACName           = "ctlog"
 	MonitoringRoleName = "prometheus-k8s-ctlog"
 
-	CertCondition   = "FulcioCertAvailable"
-	MetricsPortName = "metrics"
-	MetricsPort     = 6963
+	CertCondition    = "FulcioCertAvailable"
+	ServerPortName   = "http"
+	ServerPort       = 80
+	ServerTargetPort = 6962
+	MetricsPortName  = "metrics"
+	MetricsPort      = 6963
 )

internal/controller/ctlog/actions/deployment.go

@@ -3,6 +3,7 @@ package actions
 import (
 	"context"
 	"fmt"
+
 	cutils "github.com/securesign/operator/internal/controller/common/utils"
 
 	rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1"
@@ -39,17 +40,15 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog)
 
 	labels := constants.LabelsFor(ComponentName, DeploymentName, instance.Name)
 
-	dp, err := utils.CreateDeployment(instance, DeploymentName, RBACName, labels)
+	dp, err := utils.CreateDeployment(instance, DeploymentName, RBACName, labels, ServerTargetPort, MetricsPort)
 	if err != nil {
-		if err != nil {
-			meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
-				Type:    constants.Ready,
-				Status:  metav1.ConditionFalse,
-				Reason:  constants.Failure,
-				Message: err.Error(),
-			})
-			return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance)
-		}
+		meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
+			Type:    constants.Ready,
+			Status:  metav1.ConditionFalse,
+			Reason:  constants.Failure,
+			Message: err.Error(),
+		})
+		return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance)
 	}
 	err = cutils.SetTrustedCA(&dp.Spec.Template, cutils.TrustedCAAnnotationToReference(instance.Annotations))
 	if err != nil {

internal/controller/ctlog/actions/monitoring.go

@@ -100,7 +100,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT
 		[]monitoringv1.Endpoint{
 			{
 				Interval: monitoringv1.Duration("30s"),
-				Port:     ComponentName,
+				Port:     MetricsPortName,
 				Scheme:   "http",
 			},
 		},

internal/controller/ctlog/actions/service.go

@@ -40,13 +40,15 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog
 
 	labels := constants.LabelsFor(ComponentName, ComponentName, instance.Name)
 
-	svc := kubernetes.CreateService(instance.Namespace, ComponentName, MetricsPortName, MetricsPort, labels)
-	svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
-		Name:       "80-tcp",
-		Protocol:   corev1.ProtocolTCP,
-		Port:       80,
-		TargetPort: intstr.FromInt32(6962),
-	})
+	svc := kubernetes.CreateService(instance.Namespace, ComponentName, ServerPortName, ServerPort, ServerTargetPort, labels)
+	if instance.Spec.Monitoring.Enabled {
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+			Name:       MetricsPortName,
+			Protocol:   corev1.ProtocolTCP,
+			Port:       MetricsPort,
+			TargetPort: intstr.FromInt32(MetricsPort),
+		})
+	}
 	if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil {
 		return i.Failed(fmt.Errorf("could not set controller reference for Service: %w", err))
 	}

internal/controller/ctlog/ctlog_controller_test.go

@@ -123,7 +123,7 @@ var _ = Describe("CTlog controller", func() {
 			}).Should(Equal(constants.Pending))
 
 			By("Creating trillian service")
-			Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed())
+			Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed())
 			Eventually(func(g Gomega) string {
 				found := &v1alpha1.CTlog{}
 				g.Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed())
@@ -163,8 +163,7 @@ var _ = Describe("CTlog controller", func() {
 			Eventually(func() error {
 				return k8sClient.Get(ctx, types.NamespacedName{Name: actions.ComponentName, Namespace: Namespace}, service)
 			}).Should(Succeed())
-			Expect(service.Spec.Ports[0].Port).Should(Equal(int32(6963)))
-			Expect(service.Spec.Ports[1].Port).Should(Equal(int32(80)))
+			Expect(service.Spec.Ports[0].Port).Should(Equal(int32(80)))
 
 			By("Move to Ready phase")
 			// Workaround to succeed condition for Ready phase

internal/controller/ctlog/ctlog_hot_update_test.go

@@ -111,7 +111,7 @@ var _ = Describe("CTlog update test", func() {
 			}).Should(Succeed())
 
 			By("Creating trillian service")
-			Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed())
+			Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed())
 
 			By("Creating fulcio root cert")
 			fulcioCa := kubernetes.CreateSecret("test", Namespace,

internal/controller/ctlog/utils/ctlog_deployment.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"errors"
+	"strconv"
 
 	"github.com/securesign/operator/api/v1alpha1"
 	"github.com/securesign/operator/internal/controller/common/utils"
@@ -12,12 +13,34 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func CreateDeployment(instance *v1alpha1.CTlog, deploymentName string, sa string, labels map[string]string) (*appsv1.Deployment, error) {
+func CreateDeployment(instance *v1alpha1.CTlog, deploymentName string, sa string, labels map[string]string, serverPort, metricsPort int32) (*appsv1.Deployment, error) {
 	if instance.Status.ServerConfigRef == nil {
 		return nil, errors.New("server config name not specified")
 	}
 	replicas := int32(1)
 	// Define a new Deployment object
+
+	containerPorts := []corev1.ContainerPort{
+		{
+			ContainerPort: serverPort,
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}
+
+	appArgs := []string{
+		"--http_endpoint=0.0.0.0:" + strconv.Itoa(int(serverPort)),
+		"--log_config=/ctfe-keys/config",
+		"--alsologtostderr",
+	}
+
+	if instance.Spec.Monitoring.Enabled {
+		appArgs = append(appArgs, "--metrics_endpoint=0.0.0.0:"+strconv.Itoa(int(metricsPort)))
+		containerPorts = append(containerPorts, corev1.ContainerPort{
+			ContainerPort: metricsPort,
+			Protocol:      corev1.ProtocolTCP,
+		})
+	}
+
 	dep := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      deploymentName,
@@ -39,17 +62,12 @@ func CreateDeployment(instance *v1alpha1.CTlog, deploymentName string, sa string
 						{
 							Name:  "ctlog",
 							Image: constants.CTLogImage,
-							Args: []string{
-								"--http_endpoint=0.0.0.0:6962",
-								"--metrics_endpoint=0.0.0.0:6963",
-								"--log_config=/ctfe-keys/config",
-								"--alsologtostderr",
-							},
+							Args:  appArgs,
 							LivenessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/healthz",
-										Port: intstr.FromInt32(6962),
+										Port: intstr.FromInt32(serverPort),
 									},
 								},
 								InitialDelaySeconds: 10,
@@ -62,7 +80,7 @@ func CreateDeployment(instance *v1alpha1.CTlog, deploymentName string, sa string
 								ProbeHandler: corev1.ProbeHandler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/healthz",
-										Port: intstr.FromInt32(6962),
+										Port: intstr.FromInt32(serverPort),
 									},
 								},
 								InitialDelaySeconds: 10,
@@ -78,16 +96,7 @@ func CreateDeployment(instance *v1alpha1.CTlog, deploymentName string, sa string
 									ReadOnly:  true,
 								},
 							},
-							Ports: []corev1.ContainerPort{
-								{
-									ContainerPort: 6962,
-									Protocol:      corev1.ProtocolTCP,
-								},
-								{
-									ContainerPort: 6963,
-									Protocol:      corev1.ProtocolTCP,
-								},
-							},
+							Ports: containerPorts,
 						},
 					},
 					Volumes: []corev1.Volume{

internal/controller/fulcio/actions/constants.go

@@ -9,6 +9,11 @@ const (
 
 	CertCondition = "FulcioCertAvailable"
 
-	PortName = "metrics"
-	Port     = 2112
+	ServerPortName   = "http"
+	ServerPort       = 80
+	TargetServerPort = 5555
+	GRPCPortName     = "grpc"
+	GRPCPort         = 5554
+	MetricsPortName  = "metrics"
+	MetricsPort      = 2112
 )

internal/controller/fulcio/actions/ingress.go

@@ -43,7 +43,7 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci
 		return i.Failed(fmt.Errorf("could not find service for ingress: %w", err))
 	}
 
-	ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, instance.Spec.ExternalAccess, "80-tcp", labels)
+	ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, instance.Spec.ExternalAccess, ServerPortName, labels)
 	if err != nil {
 		return i.Failed(fmt.Errorf("could not create ingress object: %w", err))
 	}

internal/controller/fulcio/actions/monitoring.go

@@ -100,7 +100,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu
 		[]monitoringv1.Endpoint{
 			{
 				Interval: monitoringv1.Duration("30s"),
-				Port:     "fulcio-server",
+				Port:     MetricsPortName,
 				Scheme:   "http",
 			},
 		},

internal/controller/fulcio/actions/service.go

@@ -40,19 +40,23 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci
 
 	labels := constants.LabelsFor(ComponentName, DeploymentName, instance.Name)
 
-	svc := kubernetes.CreateService(instance.Namespace, DeploymentName, PortName, Port, labels)
+	svc := kubernetes.CreateService(instance.Namespace, DeploymentName, ServerPortName, ServerPort, TargetServerPort, labels)
 	svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
-		Name:       "5554-tcp",
+		Name:       GRPCPortName,
 		Protocol:   corev1.ProtocolTCP,
-		Port:       5554,
-		TargetPort: intstr.FromInt32(5554),
-	})
-	svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
-		Name:       "80-tcp",
-		Protocol:   corev1.ProtocolTCP,
-		Port:       80,
-		TargetPort: intstr.FromInt32(5555),
+		Port:       GRPCPort,
+		TargetPort: intstr.FromInt32(GRPCPort),
 	})
+
+	if instance.Spec.Monitoring.Enabled {
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+			Name:       MetricsPortName,
+			Protocol:   corev1.ProtocolTCP,
+			Port:       MetricsPort,
+			TargetPort: intstr.FromInt32(MetricsPort),
+		})
+	}
+
 	if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil {
 		return i.Failed(fmt.Errorf("could not set controller reference for Service: %w", err))
 	}

internal/controller/fulcio/fulcio_controller_test.go

@@ -209,9 +209,8 @@ var _ = Describe("Fulcio controller", func() {
 			Eventually(func() error {
 				return k8sClient.Get(ctx, types.NamespacedName{Name: actions.DeploymentName, Namespace: Namespace}, service)
 			}).Should(Succeed())
-			Expect(service.Spec.Ports[0].Port).Should(Equal(int32(2112)))
+			Expect(service.Spec.Ports[0].Port).Should(Equal(int32(80)))
 			Expect(service.Spec.Ports[1].Port).Should(Equal(int32(5554)))
-			Expect(service.Spec.Ports[2].Port).Should(Equal(int32(80)))
 
 			By("Checking if Ingress was successfully created in the reconciliation")
 			ingress := &v1.Ingress{}
@@ -220,7 +219,7 @@ var _ = Describe("Fulcio controller", func() {
 			}).Should(Succeed())
 			Expect(ingress.Spec.Rules[0].Host).Should(Equal("fulcio.localhost"))
 			Expect(ingress.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.Service.Name).Should(Equal(service.Name))
-			Expect(ingress.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.Service.Port.Name).Should(Equal("80-tcp"))
+			Expect(ingress.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.Service.Port.Name).Should(Equal(actions.ServerPortName))
 
 			By("Checking if controller will return deployment to desired state")
 			deployment = &appsv1.Deployment{}

internal/controller/fulcio/utils/fulcio_deployment.go

@@ -29,6 +29,24 @@ func CreateDeployment(instance *v1alpha1.Fulcio, deploymentName string, sa strin
 		return nil, errors.New("CA secret is not specified")
 	}
 
+	containerPorts := []corev1.ContainerPort{
+		{
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: 5555,
+		},
+		{
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: 5554,
+		},
+	}
+
+	if instance.Spec.Monitoring.Enabled {
+		containerPorts = append(containerPorts, corev1.ContainerPort{
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: 2112,
+		})
+	}
+
 	args := []string{
 		"serve",
 		"--port=5555",
@@ -95,20 +113,7 @@ func CreateDeployment(instance *v1alpha1.Fulcio, deploymentName string, sa strin
 							Image: constants.FulcioServerImage,
 							Args:  args,
 							Env:   env,
-							Ports: []corev1.ContainerPort{
-								{
-									Protocol:      corev1.ProtocolTCP,
-									ContainerPort: 5555,
-								},
-								{
-									Protocol:      corev1.ProtocolTCP,
-									ContainerPort: 5554,
-								},
-								{
-									Protocol:      corev1.ProtocolTCP,
-									ContainerPort: 2112,
-								},
-							},
+							Ports: containerPorts,
 							LivenessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									HTTPGet: &corev1.HTTPGetAction{

internal/controller/rekor/actions/constants.go

@@ -2,8 +2,11 @@ package actions
 
 const (
 	ServerDeploymentName       = "rekor-server"
-	ServerDeploymentPortName   = "metrics"
-	ServerDeploymentPort       = 2112
+	ServerDeploymentPortName   = "http"
+	ServerDeploymentPort       = 80
+	ServerTargetDeploymentPort = 3000
+	MetricsPortName            = "metrics"
+	MetricsPort                = 2112
 	RedisDeploymentName        = "rekor-redis"
 	RedisDeploymentPortName    = "resp"
 	RedisDeploymentPort        = 6379