tempostack: expose option to set RED Metrics namespace

Signed-off-by: Benedikt Bongartz <[email protected]>
grafana · Oct 29, 2024 · 6295451 · 6295451
1 parent 35b8a6f
commit 6295451
Show file tree

Hide file tree

Showing 12 changed files with 59 additions and 4 deletions.
diff --git a/.chloggen/fix_red-metrics.yaml b/.chloggen/fix_red-metrics.yaml
@@ -17,6 +17,22 @@ issues: [1072]
 subtext: |
   Fix RED metric retrieval from Prometheus by removing the default RED metric namespace introduced in Jaeger 1.62.
   The Operator overwrites the new default namespace jaeger-query takes into account using the `--prometheus.query.namespace=` flag.
+  A user can overwrite the default namespace by configuring it in the CR.
+  Example:
+  ```
+  apiVersion: tempo.grafana.com/v1alpha1
+  kind: TempoStack
+  ...
+  spec:
+    template:
+      queryFrontend:
+        jaegerQuery:
+          enabled: true
+          monitorTab:
+            enabled: true
+            prometheusEndpoint: "http://myPromInstance:9090"
+            redMetricsNamespace: "custom"
+  ```
   More details can be found here:
   - https://github.com/jaegertracing/jaeger/pull/6007
   - https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34485
diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go
@@ -689,6 +689,12 @@ type JaegerQueryMonitor struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint"
 	PrometheusEndpoint string `json:"prometheusEndpoint"`
+	// REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+	// By default no namespace is set.
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace"
+	REDMetricsNamespace string `json:"redMetricsNamespace"`
 }
 
 // IngressSpec defines Jaeger Query Ingress options.

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1070,6 +1070,10 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default no namespace is set.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -2470,6 +2470,11 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default no namespace is set.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1070,6 +1070,10 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default no namespace is set.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -2470,6 +2470,11 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default no namespace is set.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -2466,6 +2466,11 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default no namespace is set.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -999,6 +999,10 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default no namespace is set.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -999,6 +999,10 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default no namespace is set.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml
@@ -342,6 +342,7 @@ spec:                                    # TempoStackSpec defines the desired st
         monitorTab:                      # MonitorTab defines the monitor tab configuration.
           enabled: false                 # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature.
           prometheusEndpoint: ""         # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
+          redMetricsNamespace: ""        # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default no namespace is set.
         servicesQueryDuration: ""        # ServicesQueryDuration defines how long the services will be available in the services list
         tempoQuery:                      # TempoQuery defines options specific to the Tempoo Query component.
           resources:                     # Resources defines resources for this component, this will override the calculated resources derived from total

diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go
@@ -402,7 +402,7 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1.
 			// NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics".
 			// We fallback to no namespace.
 			// See https://github.com/jaegertracing/jaeger/pull/6007.
-			"--prometheus.query.namespace=",
+			fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace),
 		},
 	}
 	// If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth

diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go
@@ -621,15 +621,16 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 							JaegerQuery: v1alpha1.JaegerQuerySpec{
 								Enabled: true,
 								MonitorTab: v1alpha1.JaegerQueryMonitor{
-									Enabled:            true,
-									PrometheusEndpoint: "http://prometheus:9091",
+									Enabled:             true,
+									PrometheusEndpoint:  "http://prometheus:9091",
+									REDMetricsNamespace: "test",
 								},
 							},
 						},
 					},
 				},
 			},
-			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace="},
+			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"},
 			env:  []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}},
 		},
 		{