Skip to content

Commit

Permalink
fix: Exclude system namespaces correctly (#779)
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrkl
chrkl authored Feb 8, 2024
1 parent 9d6afc3 commit b0aa915
Show file tree
Hide file tree
Showing 7 changed files with 349 additions and 89 deletions.
35 changes: 19 additions & 16 deletions internal/fluentbit/config/builder/input.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,43 +26,46 @@ func createInputSection(pipeline *telemetryv1alpha1.LogPipeline, includePath, ex
}

func createIncludePath(pipeline *telemetryv1alpha1.LogPipeline) string {
var toInclude []string
var includePath []string

if len(pipeline.Spec.Input.Application.Namespaces.Include) == 0 {
pipeline.Spec.Input.Application.Namespaces.Include = append(pipeline.Spec.Input.Application.Namespaces.Include, "*")
includeNamespaces := []string{"*"}
if len(pipeline.Spec.Input.Application.Namespaces.Include) > 0 {
includeNamespaces = pipeline.Spec.Input.Application.Namespaces.Include
}

if len(pipeline.Spec.Input.Application.Containers.Include) == 0 {
pipeline.Spec.Input.Application.Containers.Include = append(pipeline.Spec.Input.Application.Containers.Include, "*")
includeContainers := []string{"*"}
if len(pipeline.Spec.Input.Application.Containers.Include) > 0 {
includeContainers = pipeline.Spec.Input.Application.Containers.Include
}

for _, ns := range pipeline.Spec.Input.Application.Namespaces.Include {
for _, container := range pipeline.Spec.Input.Application.Containers.Include {
toInclude = append(toInclude, makeLogPath(ns, "*", container))
for _, ns := range includeNamespaces {
for _, container := range includeContainers {
includePath = append(includePath, makeLogPath(ns, "*", container))
}
}

return strings.Join(toInclude, ",")
return strings.Join(includePath, ",")
}

func createExcludePath(pipeline *telemetryv1alpha1.LogPipeline) string {
toExclude := []string{
excludePath := []string{
makeLogPath("kyma-system", "telemetry-fluent-bit-*", "fluent-bit"),
}

if !pipeline.Spec.Input.Application.Namespaces.System {
pipeline.Spec.Input.Application.Namespaces.Exclude = append(pipeline.Spec.Input.Application.Namespaces.Exclude, namespaces.System()...)
excludeNamespaces := pipeline.Spec.Input.Application.Namespaces.Exclude
if !pipeline.Spec.Input.Application.Namespaces.System && len(pipeline.Spec.Input.Application.Namespaces.Include) == 0 && len(pipeline.Spec.Input.Application.Namespaces.Exclude) == 0 {
excludeNamespaces = namespaces.System()
}

for _, ns := range pipeline.Spec.Input.Application.Namespaces.Exclude {
toExclude = append(toExclude, makeLogPath(ns, "*", "*"))
for _, ns := range excludeNamespaces {
excludePath = append(excludePath, makeLogPath(ns, "*", "*"))
}

for _, container := range pipeline.Spec.Input.Application.Containers.Exclude {
toExclude = append(toExclude, makeLogPath("*", "*", container))
excludePath = append(excludePath, makeLogPath("*", "*", container))
}

return strings.Join(toExclude, ",")
return strings.Join(excludePath, ",")
}

func makeLogPath(namespace, pod, container string) string {
Expand Down
158 changes: 96 additions & 62 deletions internal/fluentbit/config/builder/input_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,51 @@ func TestCreateInput(t *testing.T) {
require.Equal(t, expected, actual)
}

func TestCreateInputWithIncludePath(t *testing.T) {
func TestCreateIncludeAndExcludePath(t *testing.T) {
var tests = []struct {
pipeline *telemetryv1alpha1.LogPipeline
expected []string
name string
pipeline *telemetryv1alpha1.LogPipeline
expectedIncludes []string
expectedExcludes []string
}{
{
"empty",
&telemetryv1alpha1.LogPipeline{
ObjectMeta: metav1.ObjectMeta{Name: "test-logpipeline"},
},
[]string{
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_kyma-system_*-*.log",
"/var/log/containers/*_kube-system_*-*.log",
"/var/log/containers/*_istio-system_*-*.log",
"/var/log/containers/*_compass-system_*-*.log",
},
},
{
"include system",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{},
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
System: true,
},
},
},
},
},
[]string{
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
},
},
{
"include foo namespace",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Expand All @@ -70,8 +99,12 @@ func TestCreateInputWithIncludePath(t *testing.T) {
[]string{
"/var/log/containers/*_foo_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
},
},
{
"include foo container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Expand All @@ -88,8 +121,16 @@ func TestCreateInputWithIncludePath(t *testing.T) {
[]string{
"/var/log/containers/*_*_foo-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_kyma-system_*-*.log",
"/var/log/containers/*_kube-system_*-*.log",
"/var/log/containers/*_istio-system_*-*.log",
"/var/log/containers/*_compass-system_*-*.log",
},
},
{
"include foo namespace and bar container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Expand All @@ -111,8 +152,12 @@ func TestCreateInputWithIncludePath(t *testing.T) {
[]string{
"/var/log/containers/*_foo_bar-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
},
},
{
"include foo and bar namespace, include istio-proxy container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Expand All @@ -136,55 +181,44 @@ func TestCreateInputWithIncludePath(t *testing.T) {
"/var/log/containers/*_foo_istio-proxy-*.log",
"/var/log/containers/*_bar_istio-proxy-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
},
},
}

for _, test := range tests {
actual := strings.Split(createIncludePath(test.pipeline), ",")
require.Equal(t, test.expected, actual)
}
}

func TestCreateInputWithExcludePath(t *testing.T) {
var tests = []struct {
pipeline *telemetryv1alpha1.LogPipeline
expected []string
}{
{
"exclude foo namespace",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
System: true,
Exclude: []string{
"foo",
},
},
},
},
},
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
},
},
{
&telemetryv1alpha1.LogPipeline{
ObjectMeta: metav1.ObjectMeta{Name: "test-logpipeline"},
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_kyma-system_*-*.log",
"/var/log/containers/*_kube-system_*-*.log",
"/var/log/containers/*_istio-system_*-*.log",
"/var/log/containers/*_compass-system_*-*.log",
"/var/log/containers/*_foo_*-*.log",
},
},
{
"include system, exclude foo container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
System: true,
},
Containers: telemetryv1alpha1.InputContainers{
Exclude: []string{
"foo",
},
Expand All @@ -193,17 +227,21 @@ func TestCreateInputWithExcludePath(t *testing.T) {
},
},
},
[]string{
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_foo_*-*.log",
"/var/log/containers/*_*_foo-*.log",
},
},
{
"exclude foo container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
Containers: telemetryv1alpha1.InputContainers{
Exclude: []string{
"foo",
},
Expand All @@ -212,44 +250,59 @@ func TestCreateInputWithExcludePath(t *testing.T) {
},
},
},
[]string{
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_foo_*-*.log",
"/var/log/containers/*_kyma-system_*-*.log",
"/var/log/containers/*_kube-system_*-*.log",
"/var/log/containers/*_istio-system_*-*.log",
"/var/log/containers/*_compass-system_*-*.log",
"/var/log/containers/*_*_foo-*.log",
},
},
{
"exclude foo namespace, exclude bar container",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
System: true,
Exclude: []string{
"foo",
},
},
Containers: telemetryv1alpha1.InputContainers{
Exclude: []string{
"foo",
"bar",
},
},
},
},
},
},
[]string{
"/var/log/containers/*_*_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_*_foo-*.log",
"/var/log/containers/*_foo_*-*.log",
"/var/log/containers/*_*_bar-*.log",
},
},
{
"include system and foo namespaces",
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Containers: telemetryv1alpha1.InputContainers{
Exclude: []string{
Namespaces: telemetryv1alpha1.InputNamespaces{
Include: []string{
"kyma-system",
"kube-system",
"istio-system",
"compass-system",
"foo",
},
},
Expand All @@ -258,44 +311,25 @@ func TestCreateInputWithExcludePath(t *testing.T) {
},
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_kyma-system_*-*.log",
"/var/log/containers/*_kube-system_*-*.log",
"/var/log/containers/*_istio-system_*-*.log",
"/var/log/containers/*_compass-system_*-*.log",
"/var/log/containers/*_*_foo-*.log",
},
},
{
&telemetryv1alpha1.LogPipeline{
Spec: telemetryv1alpha1.LogPipelineSpec{
Input: telemetryv1alpha1.Input{
Application: telemetryv1alpha1.ApplicationInput{
Namespaces: telemetryv1alpha1.InputNamespaces{
System: true,
Exclude: []string{
"foo",
},
},
Containers: telemetryv1alpha1.InputContainers{
Exclude: []string{
"bar",
},
},
},
},
},
"/var/log/containers/*_foo_*-*.log",
},
[]string{
"/var/log/containers/telemetry-fluent-bit-*_kyma-system_fluent-bit-*.log",
"/var/log/containers/*_foo_*-*.log",
"/var/log/containers/*_*_bar-*.log",
},
},
}

for _, test := range tests {
actual := strings.Split(createExcludePath(test.pipeline), ",")
require.Equal(t, test.expected, actual)
t.Run(test.name, func(t *testing.T) {
actualIncludes := strings.Split(createIncludePath(test.pipeline), ",")
require.Equal(t, test.expectedIncludes, actualIncludes, "Unexpected include paths for test: %s", test.name)

actualExcludes := strings.Split(createExcludePath(test.pipeline), ",")
require.Equal(t, test.expectedExcludes, actualExcludes, "Unexpected exclude paths for test: %s", test.name)
})
}
}
Loading

0 comments on commit b0aa915

Please sign in to comment.