From 418610b16cc6a3c7d547d746af805705e663f041 Mon Sep 17 00:00:00 2001 From: Jeffrey Limnardy Date: Fri, 8 Nov 2024 09:58:42 +0100 Subject: [PATCH] feat: Introduce labels in agent and gateways for usage in NetworkPolicy (#1557) Co-authored-by: Stanislav Khalash Co-authored-by: Nina Hingerl <76950046+NHingerl@users.noreply.github.com> --- docs/user/04-metrics.md | 26 +++++++++ internal/labels/labels.go | 43 ++++++++++++++ internal/labels/labels_test.go | 42 ++++++++++++++ .../reconciler/metricpipeline/reconciler.go | 11 +++- .../reconciler/tracepipeline/reconciler.go | 4 ++ internal/resources/fluentbit/resources.go | 2 + .../resources/fluentbit/resources_test.go | 7 ++- internal/resources/otelcollector/agent.go | 16 ++--- .../resources/otelcollector/agent_test.go | 16 +++-- internal/resources/otelcollector/core.go | 17 ++---- internal/resources/otelcollector/gateway.go | 58 ++++++++++--------- .../resources/otelcollector/gateway_test.go | 45 ++++++++------ internal/resources/otelcollector/rbac.go | 14 +++-- internal/resources/selfmonitor/resources.go | 25 ++++---- 14 files changed, 231 insertions(+), 95 deletions(-) create mode 100644 internal/labels/labels.go create mode 100644 internal/labels/labels_test.go diff --git a/docs/user/04-metrics.md b/docs/user/04-metrics.md index fc5497806..a7af4866a 100644 --- a/docs/user/04-metrics.md +++ b/docs/user/04-metrics.md @@ -766,6 +766,32 @@ To detect and fix such situations, check the pipeline status and check out [Trou **Remedy 2**: Define the application protocol in the Service port definition by either prefixing the port name with the protocol, like in `http-metrics` or define the `appProtocol` attribute. +**Cause 3**: A deny-all `NetworkPolicy` was created in the workload namespace, which prevents that the agent can scrape metrics from annotated workloads. + +**Remedy 3**: Create a separate `NetworkPolicy` to explicitly let the agent scrape your workload using the `telemetry.kyma-project.io/metric-scrape` label. + +For example, see the following `NetworkPolicy` configuration: +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-traffic-from-agent +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: "annotated-workload" # + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kyma-system + podSelector: + matchLabels: + telemetry.kyma-project.io/metric-scrape: "true" + policyTypes: + - Ingress +``` + ### Gateway Buffer Filling Up **Symptom**: In the MetricPipeline status, the `TelemetryFlowHealthy` condition has status **BufferFillingUp**. diff --git a/internal/labels/labels.go b/internal/labels/labels.go new file mode 100644 index 000000000..3b3041cb2 --- /dev/null +++ b/internal/labels/labels.go @@ -0,0 +1,43 @@ +package labels + +const ( + selectorLabelKey = "app.kubernetes.io/name" + traceGatewayIngestSelector = "telemetry.kyma-project.io/trace-ingest" + traceGatewayExportSelector = "telemetry.kyma-project.io/trace-export" + metricAgentScrapeSelector = "telemetry.kyma-project.io/metric-scrape" + metricGatewayIngestSelector = "telemetry.kyma-project.io/metric-ingest" + metricGatewayExportSelector = "telemetry.kyma-project.io/metric-export" + istioSidecarInjectLabel = "sidecar.istio.io/inject" +) + +func MakeDefaultLabel(baseName string) map[string]string { + return map[string]string{ + selectorLabelKey: baseName, + } +} + +func MakeMetricAgentSelectorLabel(baseName string) map[string]string { + return map[string]string{ + selectorLabelKey: baseName, + metricAgentScrapeSelector: "true", + istioSidecarInjectLabel: "true", + } +} + +func MakeMetricGatewaySelectorLabel(baseName string) map[string]string { + return map[string]string{ + selectorLabelKey: baseName, + metricGatewayIngestSelector: "true", + metricGatewayExportSelector: "true", + istioSidecarInjectLabel: "true", + } +} + +func MakeTraceGatewaySelectorLabel(baseName string) map[string]string { + return map[string]string{ + selectorLabelKey: baseName, + traceGatewayIngestSelector: "true", + traceGatewayExportSelector: "true", + istioSidecarInjectLabel: "true", + } +} diff --git a/internal/labels/labels_test.go b/internal/labels/labels_test.go new file mode 100644 index 000000000..89fad5390 --- /dev/null +++ b/internal/labels/labels_test.go @@ -0,0 +1,42 @@ +package labels + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMakeDefaultLabel(t *testing.T) { + podLabel := MakeDefaultLabel("my-pod") + require.Equal(t, map[string]string{ + "app.kubernetes.io/name": "my-pod", + }, podLabel) +} + +func TestMakeMetricAgentSelectorLabel(t *testing.T) { + metricAgentSelectorLabel := MakeMetricAgentSelectorLabel("metric-agent") + require.Equal(t, map[string]string{ + "app.kubernetes.io/name": "metric-agent", + "telemetry.kyma-project.io/metric-scrape": "true", + "sidecar.istio.io/inject": "true", + }, metricAgentSelectorLabel) +} + +func TestMakeMetricGatewaySelectorLabel(t *testing.T) { + metricGatewaySelectorLabel := MakeMetricGatewaySelectorLabel("metric-gateway") + require.Equal(t, map[string]string{ + "app.kubernetes.io/name": "metric-gateway", + "telemetry.kyma-project.io/metric-ingest": "true", + "telemetry.kyma-project.io/metric-export": "true", + "sidecar.istio.io/inject": "true", + }, metricGatewaySelectorLabel) +} +func TestMakeTraceGatewaySelectorLabel(t *testing.T) { + traceGatewaySelectorLabel := MakeTraceGatewaySelectorLabel("trace-gateway") + require.Equal(t, map[string]string{ + "app.kubernetes.io/name": "trace-gateway", + "telemetry.kyma-project.io/trace-ingest": "true", + "telemetry.kyma-project.io/trace-export": "true", + "sidecar.istio.io/inject": "true", + }, traceGatewaySelectorLabel) +} diff --git a/internal/reconciler/metricpipeline/reconciler.go b/internal/reconciler/metricpipeline/reconciler.go index 61ee87246..75e8f9e47 100644 --- a/internal/reconciler/metricpipeline/reconciler.go +++ b/internal/reconciler/metricpipeline/reconciler.go @@ -15,6 +15,7 @@ import ( telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" "github.com/kyma-project/telemetry-manager/internal/errortypes" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/metric/agent" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/metric/gateway" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/otlpexporter" @@ -266,10 +267,13 @@ func (r *Reconciler) reconcileMetricGateway(ctx context.Context, pipeline *telem allowedPorts = append(allowedPorts, ports.IstioEnvoy) } + metricGatewaySelectorLabels := labels.MakeMetricGatewaySelectorLabel(r.config.GatewayName) + opts := otelcollector.GatewayApplyOptions{ AllowedPorts: allowedPorts, CollectorConfigYAML: string(collectorConfigYAML), CollectorEnvVars: collectorEnvVars, + ComponentSelectorLabels: metricGatewaySelectorLabels, IstioEnabled: isIstioActive, IstioExcludePorts: []int32{ports.Metrics}, Replicas: r.getReplicaCountFromTelemetry(ctx), @@ -306,12 +310,15 @@ func (r *Reconciler) reconcileMetricAgents(ctx context.Context, pipeline *teleme allowedPorts = append(allowedPorts, ports.IstioEnvoy) } + metricAgentSelectorLabels := labels.MakeMetricAgentSelectorLabel(r.config.AgentName) + if err := r.agentApplierDeleter.ApplyResources( ctx, k8sutils.NewOwnerReferenceSetter(r.Client, pipeline), otelcollector.AgentApplyOptions{ - AllowedPorts: allowedPorts, - CollectorConfigYAML: string(agentConfigYAML), + AllowedPorts: allowedPorts, + CollectorConfigYAML: string(agentConfigYAML), + ComponentSelectorLabels: metricAgentSelectorLabels, }, ); err != nil { return fmt.Errorf("failed to apply agent resources: %w", err) diff --git a/internal/reconciler/tracepipeline/reconciler.go b/internal/reconciler/tracepipeline/reconciler.go index 2e31016b5..81855b58e 100644 --- a/internal/reconciler/tracepipeline/reconciler.go +++ b/internal/reconciler/tracepipeline/reconciler.go @@ -31,6 +31,7 @@ import ( telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" "github.com/kyma-project/telemetry-manager/internal/errortypes" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/otlpexporter" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config/trace/gateway" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" @@ -245,10 +246,13 @@ func (r *Reconciler) reconcileTraceGateway(ctx context.Context, pipeline *teleme allowedPorts = append(allowedPorts, ports.IstioEnvoy) } + traceGatewaySelectorLabels := labels.MakeTraceGatewaySelectorLabel(r.config.TraceGatewayName) + opts := otelcollector.GatewayApplyOptions{ AllowedPorts: allowedPorts, CollectorConfigYAML: string(collectorConfigYAML), CollectorEnvVars: collectorEnvVars, + ComponentSelectorLabels: traceGatewaySelectorLabels, IstioEnabled: isIstioActive, IstioExcludePorts: []int32{ports.Metrics}, Replicas: r.getReplicaCountFromTelemetry(ctx), diff --git a/internal/resources/fluentbit/resources.go b/internal/resources/fluentbit/resources.go index ece9f20db..5a1577450 100644 --- a/internal/resources/fluentbit/resources.go +++ b/internal/resources/fluentbit/resources.go @@ -19,6 +19,7 @@ import ( const checksumAnnotationKey = "checksum/logpipeline-config" const istioExcludeInboundPorts = "traffic.sidecar.istio.io/excludeInboundPorts" +const fluentbitExportSelector = "telemetry.kyma-project.io/log-export" type DaemonSetConfig struct { FluentBitImage string @@ -61,6 +62,7 @@ func MakeDaemonSet(name types.NamespacedName, checksum string, dsConfig DaemonSe podLabels := Labels() podLabels["sidecar.istio.io/inject"] = "true" + podLabels[fluentbitExportSelector] = "true" return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{}, diff --git a/internal/resources/fluentbit/resources_test.go b/internal/resources/fluentbit/resources_test.go index 676ff7087..b9963941f 100644 --- a/internal/resources/fluentbit/resources_test.go +++ b/internal/resources/fluentbit/resources_test.go @@ -39,9 +39,10 @@ func TestMakeDaemonSet(t *testing.T) { "app.kubernetes.io/instance": "telemetry", }, daemonSet.Spec.Selector.MatchLabels) require.Equal(t, map[string]string{ - "app.kubernetes.io/name": "fluent-bit", - "app.kubernetes.io/instance": "telemetry", - "sidecar.istio.io/inject": "true", + "app.kubernetes.io/name": "fluent-bit", + "app.kubernetes.io/instance": "telemetry", + "sidecar.istio.io/inject": "true", + "telemetry.kyma-project.io/log-export": "true", }, daemonSet.Spec.Template.ObjectMeta.Labels) require.NotEmpty(t, daemonSet.Spec.Template.Spec.Containers[0].EnvFrom) require.NotNil(t, daemonSet.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness probe must be defined") diff --git a/internal/resources/otelcollector/agent.go b/internal/resources/otelcollector/agent.go index d86aed21d..221097524 100644 --- a/internal/resources/otelcollector/agent.go +++ b/internal/resources/otelcollector/agent.go @@ -16,6 +16,7 @@ import ( "github.com/kyma-project/telemetry-manager/internal/configchecksum" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common" @@ -32,8 +33,9 @@ type AgentApplierDeleter struct { } type AgentApplyOptions struct { - AllowedPorts []int32 - CollectorConfigYAML string + AllowedPorts []int32 + CollectorConfigYAML string + ComponentSelectorLabels map[string]string } func (aad *AgentApplierDeleter) ApplyResources(ctx context.Context, c client.Client, opts AgentApplyOptions) error { @@ -49,7 +51,7 @@ func (aad *AgentApplierDeleter) ApplyResources(ctx context.Context, c client.Cli } configChecksum := configchecksum.Calculate([]corev1.ConfigMap{*configMap}, []corev1.Secret{}) - if err := k8sutils.CreateOrUpdateDaemonSet(ctx, c, aad.makeAgentDaemonSet(configChecksum)); err != nil { + if err := k8sutils.CreateOrUpdateDaemonSet(ctx, c, aad.makeAgentDaemonSet(configChecksum, opts)); err != nil { return fmt.Errorf("failed to create daemonset: %w", err) } @@ -83,10 +85,8 @@ func (aad *AgentApplierDeleter) DeleteResources(ctx context.Context, c client.Cl return allErrors } -func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string) *appsv1.DaemonSet { - selectorLabels := defaultLabels(aad.Config.BaseName) - podLabels := maps.Clone(selectorLabels) - podLabels["sidecar.istio.io/inject"] = "true" +func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string, opts AgentApplyOptions) *appsv1.DaemonSet { + selectorLabels := labels.MakeDefaultLabel(aad.Config.BaseName) annotations := map[string]string{"checksum/config": configChecksum} maps.Copy(annotations, makeIstioTLSPodAnnotations(IstioCertPath)) @@ -123,7 +123,7 @@ func (aad *AgentApplierDeleter) makeAgentDaemonSet(configChecksum string) *appsv }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, + Labels: opts.ComponentSelectorLabels, Annotations: annotations, }, Spec: podSpec, diff --git a/internal/resources/otelcollector/agent_test.go b/internal/resources/otelcollector/agent_test.go index e6acb6979..09dc1009e 100644 --- a/internal/resources/otelcollector/agent_test.go +++ b/internal/resources/otelcollector/agent_test.go @@ -14,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kyma-project/telemetry-manager/internal/labels" ) var ( @@ -37,8 +39,9 @@ func TestApplyAgentResources(t *testing.T) { } err := sut.ApplyResources(ctx, client, AgentApplyOptions{ - AllowedPorts: []int32{5555, 6666}, - CollectorConfigYAML: agentCfg, + AllowedPorts: []int32{5555, 6666}, + CollectorConfigYAML: agentCfg, + ComponentSelectorLabels: labels.MakeMetricAgentSelectorLabel(agentName), }) require.NoError(t, err) @@ -205,8 +208,9 @@ func TestApplyAgentResources(t *testing.T) { "app.kubernetes.io/name": agentName, }, ds.Spec.Selector.MatchLabels, "must have expected daemonset selector labels") require.Equal(t, map[string]string{ - "app.kubernetes.io/name": agentName, - "sidecar.istio.io/inject": "true", + "app.kubernetes.io/name": agentName, + "sidecar.istio.io/inject": "true", + "telemetry.kyma-project.io/metric-scrape": "true", }, ds.Spec.Template.ObjectMeta.Labels, "must have expected pod labels") // annotations @@ -323,7 +327,7 @@ func createAgentRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: agentName, Namespace: agentNamespace, - Labels: defaultLabels(agentName), + Labels: labels.MakeDefaultLabel(agentName), }, Rules: []rbacv1.PolicyRule{ { @@ -338,7 +342,7 @@ func createAgentRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: agentName, Namespace: agentNamespace, - Labels: defaultLabels(agentName), + Labels: labels.MakeDefaultLabel(agentName), }, Subjects: []rbacv1.Subject{{Name: agentName, Namespace: agentNamespace, Kind: rbacv1.ServiceAccountKind}}, RoleRef: rbacv1.RoleRef{ diff --git a/internal/resources/otelcollector/core.go b/internal/resources/otelcollector/core.go index 155568ce6..f66f03e68 100644 --- a/internal/resources/otelcollector/core.go +++ b/internal/resources/otelcollector/core.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common" ) @@ -64,7 +65,7 @@ func applyCommonResources(ctx context.Context, c client.Client, name types.Names return fmt.Errorf("failed to create metrics service: %w", err) } - if err := k8sutils.CreateOrUpdateNetworkPolicy(ctx, c, commonresources.MakeNetworkPolicy(name, allowedPorts, defaultLabels(name.Name))); err != nil { + if err := k8sutils.CreateOrUpdateNetworkPolicy(ctx, c, commonresources.MakeNetworkPolicy(name, allowedPorts, labels.MakeDefaultLabel(name.Name))); err != nil { return fmt.Errorf("failed to create network policy: %w", err) } @@ -118,18 +119,12 @@ func deleteCommonResources(ctx context.Context, c client.Client, name types.Name return allErrors } -func defaultLabels(baseName string) map[string]string { - return map[string]string{ - "app.kubernetes.io/name": baseName, - } -} - func makeServiceAccount(name types.NamespacedName) *corev1.ServiceAccount { serviceAccount := corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, } @@ -141,7 +136,7 @@ func makeConfigMap(name types.NamespacedName, collectorConfigYAML string) *corev ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Data: map[string]string{ configMapKey: collectorConfigYAML, @@ -154,14 +149,14 @@ func makeSecret(name types.NamespacedName, secretData map[string][]byte) *corev1 ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Data: secretData, } } func makeMetricsService(name types.NamespacedName) *corev1.Service { - labels := defaultLabels(name.Name) + labels := labels.MakeDefaultLabel(name.Name) selectorLabels := make(map[string]string) maps.Copy(selectorLabels, labels) labels["telemetry.kyma-project.io/self-monitor"] = "enabled" diff --git a/internal/resources/otelcollector/gateway.go b/internal/resources/otelcollector/gateway.go index 116dd2407..92795a811 100644 --- a/internal/resources/otelcollector/gateway.go +++ b/internal/resources/otelcollector/gateway.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "maps" "strings" istiosecurityv1 "istio.io/api/security/v1" @@ -21,6 +20,7 @@ import ( "github.com/kyma-project/telemetry-manager/internal/configchecksum" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" "github.com/kyma-project/telemetry-manager/internal/otelcollector/config" "github.com/kyma-project/telemetry-manager/internal/otelcollector/ports" commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common" @@ -32,11 +32,12 @@ type GatewayApplierDeleter struct { } type GatewayApplyOptions struct { - AllowedPorts []int32 - CollectorConfigYAML string - CollectorEnvVars map[string][]byte - IstioEnabled bool - IstioExcludePorts []int32 + AllowedPorts []int32 + CollectorConfigYAML string + CollectorEnvVars map[string][]byte + ComponentSelectorLabels map[string]string + IstioEnabled bool + IstioExcludePorts []int32 // Replicas specifies the number of gateway replicas. Replicas int32 // ResourceRequirementsMultiplier is a coefficient affecting the CPU and memory resource limits for each replica. @@ -125,29 +126,15 @@ func (gad *GatewayApplierDeleter) DeleteResources(ctx context.Context, c client. } func (gad *GatewayApplierDeleter) makeGatewayDeployment(configChecksum string, opts GatewayApplyOptions) *appsv1.Deployment { - selectorLabels := defaultLabels(gad.Config.BaseName) - podLabels := maps.Clone(selectorLabels) - podLabels["sidecar.istio.io/inject"] = fmt.Sprintf("%t", opts.IstioEnabled) + selectorLabels := labels.MakeDefaultLabel(gad.Config.BaseName) - annotations := map[string]string{"checksum/config": configChecksum} - - if opts.IstioEnabled { - var excludeInboundPorts []string - for _, p := range opts.IstioExcludePorts { - excludeInboundPorts = append(excludeInboundPorts, fmt.Sprintf("%d", p)) - } - - annotations["traffic.sidecar.istio.io/excludeInboundPorts"] = strings.Join(excludeInboundPorts, ", ") - // When a workload is outside the istio mesh and communicates with pod in service mesh, the envoy proxy does not - // preserve the source IP and destination IP. To preserve source/destination IP we need TPROXY interception mode. - // More info: https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#ProxyConfig-InboundInterceptionMode - annotations["sidecar.istio.io/interceptionMode"] = "TPROXY" - } + annotations := gad.makeAnnotations(configChecksum, opts) resources := gad.makeGatewayResourceRequirements(opts) affinity := makePodAffinity(selectorLabels) deploymentConfig := gad.Config.Deployment + podSpec := makePodSpec( gad.Config.BaseName, deploymentConfig.Image, @@ -172,7 +159,7 @@ func (gad *GatewayApplierDeleter) makeGatewayDeployment(configChecksum string, o }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, + Labels: opts.ComponentSelectorLabels, Annotations: annotations, }, Spec: podSpec, @@ -238,7 +225,7 @@ func makePodAffinity(labels map[string]string) corev1.Affinity { } func (gad *GatewayApplierDeleter) makeOTLPService() *corev1.Service { - labels := defaultLabels(gad.Config.BaseName) + labels := labels.MakeDefaultLabel(gad.Config.BaseName) return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -268,7 +255,7 @@ func (gad *GatewayApplierDeleter) makeOTLPService() *corev1.Service { } func (gad *GatewayApplierDeleter) makePeerAuthentication() *istiosecurityclientv1.PeerAuthentication { - labels := defaultLabels(gad.Config.BaseName) + labels := labels.MakeDefaultLabel(gad.Config.BaseName) return &istiosecurityclientv1.PeerAuthentication{ ObjectMeta: metav1.ObjectMeta{ @@ -282,3 +269,22 @@ func (gad *GatewayApplierDeleter) makePeerAuthentication() *istiosecurityclientv }, } } + +func (gad *GatewayApplierDeleter) makeAnnotations(configChecksum string, opts GatewayApplyOptions) map[string]string { + annotations := map[string]string{"checksum/config": configChecksum} + + if opts.IstioEnabled { + var excludeInboundPorts []string + for _, p := range opts.IstioExcludePorts { + excludeInboundPorts = append(excludeInboundPorts, fmt.Sprintf("%d", p)) + } + + annotations["traffic.sidecar.istio.io/excludeInboundPorts"] = strings.Join(excludeInboundPorts, ", ") + // When a workload is outside the istio mesh and communicates with pod in service mesh, the envoy proxy does not + // preserve the source IP and destination IP. To preserve source/destination IP we need TPROXY interception mode. + // More info: https://istio.io/latest/docs/reference/config/istio.mesh.v1alpha1/#ProxyConfig-InboundInterceptionMode + annotations["sidecar.istio.io/interceptionMode"] = "TPROXY" + } + + return annotations +} diff --git a/internal/resources/otelcollector/gateway_test.go b/internal/resources/otelcollector/gateway_test.go index 332e3b476..227d72aa5 100644 --- a/internal/resources/otelcollector/gateway_test.go +++ b/internal/resources/otelcollector/gateway_test.go @@ -19,6 +19,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kyma-project/telemetry-manager/internal/labels" ) var ( @@ -54,6 +56,7 @@ func TestApplyGatewayResources(t *testing.T) { AllowedPorts: []int32{5555, 6666}, CollectorConfigYAML: gatewayCfg, CollectorEnvVars: envVars, + ComponentSelectorLabels: labels.MakeTraceGatewaySelectorLabel(gatewayName), Replicas: replicas, ResourceRequirementsMultiplier: 1, }) @@ -278,8 +281,10 @@ func TestApplyGatewayResources(t *testing.T) { "app.kubernetes.io/name": gatewayName, }, dep.Spec.Selector.MatchLabels, "must have expected deployment selector labels") require.Equal(t, map[string]string{ - "app.kubernetes.io/name": gatewayName, - "sidecar.istio.io/inject": "false", + "app.kubernetes.io/name": gatewayName, + "sidecar.istio.io/inject": "true", + "telemetry.kyma-project.io/trace-ingest": "true", + "telemetry.kyma-project.io/trace-export": "true", }, dep.Spec.Template.ObjectMeta.Labels, "must have expected pod labels") // annotations @@ -377,11 +382,12 @@ func TestApplyGatewayResourcesWithIstioEnabled(t *testing.T) { } err := sut.ApplyResources(ctx, client, GatewayApplyOptions{ - CollectorConfigYAML: gatewayCfg, - CollectorEnvVars: envVars, - IstioEnabled: true, - IstioExcludePorts: []int32{1111, 2222}, - Replicas: replicas, + CollectorConfigYAML: gatewayCfg, + CollectorEnvVars: envVars, + ComponentSelectorLabels: labels.MakeTraceGatewaySelectorLabel(gatewayName), + IstioEnabled: true, + IstioExcludePorts: []int32{1111, 2222}, + Replicas: replicas, }) require.NoError(t, err) @@ -405,8 +411,10 @@ func TestApplyGatewayResourcesWithIstioEnabled(t *testing.T) { require.Equal(t, replicas, *dep.Spec.Replicas) require.Equal(t, map[string]string{ - "app.kubernetes.io/name": gatewayName, - "sidecar.istio.io/inject": "true", + "app.kubernetes.io/name": gatewayName, + "telemetry.kyma-project.io/trace-ingest": "true", + "telemetry.kyma-project.io/trace-export": "true", + "sidecar.istio.io/inject": "true", }, dep.Spec.Template.ObjectMeta.Labels, "must have expected pod labels") // annotations @@ -431,11 +439,12 @@ func TestDeleteGatewayResources(t *testing.T) { // Create gateway resources before testing deletion err := sut.ApplyResources(ctx, client, GatewayApplyOptions{ - CollectorConfigYAML: gatewayCfg, - CollectorEnvVars: envVars, - IstioEnabled: true, - IstioExcludePorts: []int32{1111, 2222}, - Replicas: replicas, + CollectorConfigYAML: gatewayCfg, + CollectorEnvVars: envVars, + ComponentSelectorLabels: labels.MakeTraceGatewaySelectorLabel(gatewayName), + IstioEnabled: true, + IstioExcludePorts: []int32{1111, 2222}, + Replicas: replicas, }) require.NoError(t, err) @@ -542,7 +551,7 @@ func createGatewayRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: gatewayName, Namespace: gatewayNamespace, - Labels: defaultLabels(gatewayName), + Labels: labels.MakeDefaultLabel(gatewayName), }, Rules: []rbacv1.PolicyRule{ { @@ -557,7 +566,7 @@ func createGatewayRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: gatewayName, Namespace: gatewayNamespace, - Labels: defaultLabels(gatewayName), + Labels: labels.MakeDefaultLabel(gatewayName), }, Subjects: []rbacv1.Subject{{Name: gatewayName, Namespace: gatewayNamespace, Kind: rbacv1.ServiceAccountKind}}, RoleRef: rbacv1.RoleRef{ @@ -571,7 +580,7 @@ func createGatewayRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: gatewayName, Namespace: gatewayNamespace, - Labels: defaultLabels(gatewayName), + Labels: labels.MakeDefaultLabel(gatewayName), }, Rules: []rbacv1.PolicyRule{ { @@ -586,7 +595,7 @@ func createGatewayRBAC() Rbac { ObjectMeta: metav1.ObjectMeta{ Name: gatewayName, Namespace: gatewayNamespace, - Labels: defaultLabels(gatewayName), + Labels: labels.MakeDefaultLabel(gatewayName), }, Subjects: []rbacv1.Subject{{Name: gatewayName, Namespace: gatewayNamespace, Kind: rbacv1.ServiceAccountKind}}, RoleRef: rbacv1.RoleRef{ diff --git a/internal/resources/otelcollector/rbac.go b/internal/resources/otelcollector/rbac.go index 3e71eb944..273e60450 100644 --- a/internal/resources/otelcollector/rbac.go +++ b/internal/resources/otelcollector/rbac.go @@ -4,6 +4,8 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + + "github.com/kyma-project/telemetry-manager/internal/labels" ) type Rbac struct { @@ -45,7 +47,7 @@ func makeTraceGatewayClusterRole(name types.NamespacedName) *rbacv1.ClusterRole ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Rules: []rbacv1.PolicyRule{ { @@ -67,7 +69,7 @@ func makeMetricAgentClusterRole(name types.NamespacedName) *rbacv1.ClusterRole { ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Rules: []rbacv1.PolicyRule{ { @@ -114,7 +116,7 @@ func makeMetricGatewayClusterRole(name types.NamespacedName) *rbacv1.ClusterRole ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Rules: []rbacv1.PolicyRule{ { @@ -158,7 +160,7 @@ func makeClusterRoleBinding(name types.NamespacedName) *rbacv1.ClusterRoleBindin ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Subjects: []rbacv1.Subject{{Name: name.Name, Namespace: name.Namespace, Kind: rbacv1.ServiceAccountKind}}, RoleRef: rbacv1.RoleRef{ @@ -174,7 +176,7 @@ func makeMetricRole(name types.NamespacedName) *rbacv1.Role { ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Rules: []rbacv1.PolicyRule{ { @@ -190,7 +192,7 @@ func makeMetricRoleBinding(name types.NamespacedName) *rbacv1.RoleBinding { ObjectMeta: metav1.ObjectMeta{ Name: name.Name, Namespace: name.Namespace, - Labels: defaultLabels(name.Name), + Labels: labels.MakeDefaultLabel(name.Name), }, Subjects: []rbacv1.Subject{ { diff --git a/internal/resources/selfmonitor/resources.go b/internal/resources/selfmonitor/resources.go index 64cc81f8c..fa507f9fd 100644 --- a/internal/resources/selfmonitor/resources.go +++ b/internal/resources/selfmonitor/resources.go @@ -17,6 +17,7 @@ import ( "github.com/kyma-project/telemetry-manager/internal/configchecksum" "github.com/kyma-project/telemetry-manager/internal/k8sutils" + "github.com/kyma-project/telemetry-manager/internal/labels" commonresources "github.com/kyma-project/telemetry-manager/internal/resources/common" "github.com/kyma-project/telemetry-manager/internal/selfmonitor/ports" ) @@ -127,7 +128,7 @@ func (ad *ApplierDeleter) makeServiceAccount() *corev1.ServiceAccount { ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, } @@ -139,7 +140,7 @@ func (ad *ApplierDeleter) makeRole() *rbacv1.Role { ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, Rules: []rbacv1.PolicyRule{ { @@ -158,7 +159,7 @@ func (ad *ApplierDeleter) makeRoleBinding() *rbacv1.RoleBinding { ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, Subjects: []rbacv1.Subject{{Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, Kind: rbacv1.ServiceAccountKind}}, RoleRef: rbacv1.RoleRef{ @@ -178,11 +179,11 @@ func (ad *ApplierDeleter) makeNetworkPolicy() *networkingv1.NetworkPolicy { ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{ - MatchLabels: ad.defaultLabels(), + MatchLabels: labels.MakeDefaultLabel(ad.Config.BaseName), }, PolicyTypes: []networkingv1.PolicyType{ networkingv1.PolicyTypeIngress, @@ -238,7 +239,7 @@ func (ad *ApplierDeleter) makeConfigMap(prometheusConfigFileName, prometheusConf ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, Data: map[string]string{ prometheusConfigFileName: prometheusConfigYAML, @@ -250,7 +251,7 @@ func (ad *ApplierDeleter) makeConfigMap(prometheusConfigFileName, prometheusConf func (ad *ApplierDeleter) makeDeployment(configChecksum, configPath, configFile string) *appsv1.Deployment { var replicas int32 = 1 - selectorLabels := ad.defaultLabels() + selectorLabels := labels.MakeDefaultLabel(ad.Config.BaseName) podLabels := maps.Clone(selectorLabels) podLabels["sidecar.istio.io/inject"] = "false" @@ -284,12 +285,6 @@ func (ad *ApplierDeleter) makeDeployment(configChecksum, configPath, configFile } } -func (ad *ApplierDeleter) defaultLabels() map[string]string { - return map[string]string{ - "app.kubernetes.io/name": ad.Config.BaseName, - } -} - func makePodSpec(baseName, image, configPath, configFile string, opts ...commonresources.PodSpecOption) corev1.PodSpec { var defaultMode int32 = 420 @@ -405,7 +400,7 @@ func (ad *ApplierDeleter) makeService(port int32) *corev1.Service { ObjectMeta: metav1.ObjectMeta{ Name: ad.Config.BaseName, Namespace: ad.Config.Namespace, - Labels: ad.defaultLabels(), + Labels: labels.MakeDefaultLabel(ad.Config.BaseName), }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -416,7 +411,7 @@ func (ad *ApplierDeleter) makeService(port int32) *corev1.Service { TargetPort: intstr.FromInt32(port), }, }, - Selector: ad.defaultLabels(), + Selector: labels.MakeDefaultLabel(ad.Config.BaseName), Type: corev1.ServiceTypeClusterIP, }, }