Add e2e tests for LogPipeline namespace selection

test/e2e/logs_exclude_namespaces_test.go

@@ -0,0 +1,102 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"net/http"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s"
+	. "github.com/kyma-project/telemetry-manager/test/testkit/matchers/log"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/loggen"
+	"github.com/kyma-project/telemetry-manager/test/testkit/periodic"
+	"github.com/kyma-project/telemetry-manager/test/testkit/verifiers"
+)
+
+var _ = Describe("Logs Exclude Namespace", Label("logs"), Ordered, func() {
+	const (
+		mockNs          = "log-exclude-namespace-mocks"
+		mockBackendName = "log-receiver-exclude-namespace"
+		logProducerName = "log-producer-exclude-namespace"
+		pipelineName    = "pipeline-exclude-namespace-test"
+	)
+	var telemetryExportURL string
+
+	makeResources := func() []client.Object {
+		var objs []client.Object
+		objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())
+
+		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeLogs)
+		mockLogProducer := loggen.New(logProducerName, mockNs)
+		objs = append(objs, mockBackend.K8sObjects()...)
+		objs = append(objs, mockLogProducer.K8sObject(kitk8s.WithLabel("app", "logging-exclude-namespace")))
+		telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient)
+
+		logPipeline := kitk8s.NewLogPipeline(pipelineName).
+			WithSecretKeyRef(mockBackend.HostSecretRef()).
+			WithHTTPOutput().
+			WithExcludeNamespaces([]string{"kyma-system"})
+		objs = append(objs, logPipeline.K8sObject())
+
+		return objs
+	}
+
+	Context("Before deploying a logpipeline", func() {
+		It("Should have a healthy webhook", func() {
+			verifiers.WebhookShouldBeHealthy(ctx, k8sClient)
+		})
+	})
+
+	Context("When a logpipeline that excludes containers exists", Ordered, func() {
+		BeforeAll(func() {
+			k8sObjects := makeResources()
+			DeferCleanup(func() {
+				Expect(kitk8s.DeleteObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
+			})
+			Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
+		})
+
+		It("Should have a running logpipeline", func() {
+			verifiers.LogPipelineShouldBeRunning(ctx, k8sClient, pipelineName)
+		})
+
+		It("Should have a log backend running", func() {
+			verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: mockBackendName})
+		})
+
+		It("Should have a log producer running", func() {
+			verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: logProducerName})
+		})
+
+		It("Should have any logs in the backend", func() {
+			Eventually(func(g Gomega) {
+				resp, err := proxyClient.Get(telemetryExportURL)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
+				g.Expect(resp).To(HaveHTTPBody(
+					ContainLd(
+						SatisfyAll(
+							ContainLogRecord(WithNamespace(Equal("kube-system"))),
+							ContainLogRecord(WithNamespace(Equal(mockNs))),
+						)),
+				))
+			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
+		})
+
+		It("Should have no kyma-system logs in the backend", func() {
+			Consistently(func(g Gomega) {
+				resp, err := proxyClient.Get(telemetryExportURL)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
+				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
+					WithNamespace(Equal("kyma-system"))))),
+				))
+			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
+		})
+	})
+})

test/e2e/logs_include_namespaces_test.go

@@ -0,0 +1,104 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"net/http"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s"
+	. "github.com/kyma-project/telemetry-manager/test/testkit/matchers/log"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/backend"
+	"github.com/kyma-project/telemetry-manager/test/testkit/mocks/loggen"
+	"github.com/kyma-project/telemetry-manager/test/testkit/periodic"
+	"github.com/kyma-project/telemetry-manager/test/testkit/verifiers"
+)
+
+var _ = Describe("Logs Include Namespaces", Label("logs"), Ordered, func() {
+	const (
+		mockNs          = "log-include-namespaces-mocks"
+		systemNamespace = "kyma-system"
+		mockBackendName = "log-receiver-include-namespaces"
+		logProducerName = "log-producer-include-namespaces"
+		pipelineName    = "pipeline-include-namespaces-test"
+	)
+	var telemetryExportURL string
+
+	makeResources := func() []client.Object {
+		var objs []client.Object
+		objs = append(objs, kitk8s.NewNamespace(mockNs).K8sObject())
+
+		mockBackend := backend.New(mockBackendName, mockNs, backend.SignalTypeLogs)
+		mockLogProducer := loggen.New(logProducerName, mockNs)
+		objs = append(objs, mockBackend.K8sObjects()...)
+		objs = append(objs, mockLogProducer.K8sObject(kitk8s.WithLabel("app", "logging-include-namespaces")))
+		telemetryExportURL = mockBackend.TelemetryExportURL(proxyClient)
+
+		logPipeline := kitk8s.NewLogPipeline(pipelineName).
+			WithSecretKeyRef(mockBackend.HostSecretRef()).
+			WithHTTPOutput().
+			WithIncludeNamespaces([]string{"kyma-system", mockNs}).
+			WithExcludeContainers([]string{logProducerName})
+		objs = append(objs, logPipeline.K8sObject())
+
+		return objs
+	}
+
+	Context("Before deploying a logpipeline", func() {
+		It("Should have a healthy webhook", func() {
+			verifiers.WebhookShouldBeHealthy(ctx, k8sClient)
+		})
+	})
+
+	Context("When a logpipeline that includes namespaces exists", Ordered, func() {
+		BeforeAll(func() {
+			k8sObjects := makeResources()
+			DeferCleanup(func() {
+				Expect(kitk8s.DeleteObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
+			})
+			Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed())
+		})
+
+		It("Should have a running logpipeline", func() {
+			verifiers.LogPipelineShouldBeRunning(ctx, k8sClient, pipelineName)
+		})
+
+		It("Should have a log backend running", func() {
+			verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: mockBackendName})
+		})
+
+		It("Should have a log producer running", func() {
+			verifiers.DeploymentShouldBeReady(ctx, k8sClient, types.NamespacedName{Namespace: mockNs, Name: logProducerName})
+		})
+
+		It("Should have logs from expected namespaces", func() {
+			Eventually(func(g Gomega) {
+				resp, err := proxyClient.Get(telemetryExportURL)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
+				g.Expect(resp).To(HaveHTTPBody(
+					ContainLd(
+						SatisfyAll(
+							ContainLogRecord(WithNamespace(Equal(systemNamespace))),
+							ContainLogRecord(WithNamespace(Equal(mockNs))),
+						)),
+				))
+			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
+		})
+
+		It("Should have no no logs from kube-system in the backend", func() {
+			Consistently(func(g Gomega) {
+				resp, err := proxyClient.Get(telemetryExportURL)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
+				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
+					WithNamespace(Equal("kube-system"))))),
+				))
+			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
+		})
+	})
+})

test/testkit/k8s/log_pipeline.go

@@ -15,6 +15,8 @@ type LogPipeline struct {
 	name              string
 	secretKeyRef      *telemetryv1alpha1.SecretKeyRef
 	systemNamespaces  bool
+	includeNamespaces []string
+	excludeNamespaces []string
 	includeContainers []string
 	excludeContainers []string
 	keepAnnotations   bool
@@ -43,6 +45,16 @@ func (p *LogPipeline) WithSystemNamespaces(enable bool) *LogPipeline {
 	return p
 }
 
+func (p *LogPipeline) WithIncludeNamespaces(namespaces []string) *LogPipeline {
+	p.includeNamespaces = namespaces
+	return p
+}
+
+func (p *LogPipeline) WithExcludeNamespaces(namespaces []string) *LogPipeline {
+	p.excludeNamespaces = namespaces
+	return p
+}
+
 func (p *LogPipeline) WithIncludeContainers(names []string) *LogPipeline {
 	p.includeContainers = names
 	return p
@@ -154,7 +166,9 @@ func (p *LogPipeline) K8sObject() *telemetryv1alpha1.LogPipeline {
 			Input: telemetryv1alpha1.Input{
 				Application: telemetryv1alpha1.ApplicationInput{
 					Namespaces: telemetryv1alpha1.InputNamespaces{
-						System: p.systemNamespaces,
+						System:  p.systemNamespaces,
+						Include: p.includeNamespaces,
+						Exclude: p.excludeNamespaces,
 					},
 					Containers: telemetryv1alpha1.InputContainers{
 						Include: p.includeContainers,

test/testkit/matchers/log/log_matchers.go

@@ -30,11 +30,6 @@ func ContainLd(matcher types.GomegaMatcher) types.GomegaMatcher {
 	return WithLds(gomega.ContainElement(matcher))
 }
 
-// ConsistOfLds is an alias for WithLds(gomega.ConsistOf()).
-func ConsistOfLds(matcher types.GomegaMatcher) types.GomegaMatcher {
-	return WithLds(gomega.ConsistOf(matcher))
-}
-
 func WithLogRecords(matcher types.GomegaMatcher) types.GomegaMatcher {
 	return gomega.WithTransform(func(ld plog.Logs) ([]plog.LogRecord, error) {
 		return getLogRecords(ld), nil
@@ -46,11 +41,6 @@ func ContainLogRecord(matcher types.GomegaMatcher) types.GomegaMatcher {
 	return WithLogRecords(gomega.ContainElement(matcher))
 }
 
-// ConsistOfLogRecords is an alias for WithLogRecords(gomega.ConsistOf()).
-func ConsistOfLogRecords(matcher types.GomegaMatcher) types.GomegaMatcher {
-	return WithLogRecords(gomega.ConsistOf(matcher))
-}
-
 func WithContainerName(matcher types.GomegaMatcher) types.GomegaMatcher {
 	return gomega.WithTransform(func(lr plog.LogRecord) string {
 		kubernetesAttrs := getKubernetesAttributes(lr)
@@ -63,6 +53,18 @@ func WithContainerName(matcher types.GomegaMatcher) types.GomegaMatcher {
 	}, matcher)
 }
 
+func WithNamespace(matcher types.GomegaMatcher) types.GomegaMatcher {
+	return gomega.WithTransform(func(lr plog.LogRecord) string {
+		kubernetesAttrs := getKubernetesAttributes(lr)
+		namespaceName, hasNamespaceName := kubernetesAttrs.Get("namespace_name")
+		if !hasNamespaceName || namespaceName.Type() != pcommon.ValueTypeStr {
+			return ""
+		}
+
+		return namespaceName.Str()
+	}, matcher)
+}
+
 func WithPodName(matcher types.GomegaMatcher) types.GomegaMatcher {
 	return gomega.WithTransform(func(lr plog.LogRecord) string {
 		kubernetesAttrs := getKubernetesAttributes(lr)