From 8b2a27c825552b62b501197bc6c9ee0b17fac42b Mon Sep 17 00:00:00 2001 From: alejandroEsc Date: Fri, 9 Oct 2020 13:24:49 -0700 Subject: [PATCH 1/3] fix: fluentbit should not keep retrying indefinitely --- addons/fluentbit/fluentbit.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/addons/fluentbit/fluentbit.yaml b/addons/fluentbit/fluentbit.yaml index 25010c14..5b025546 100644 --- a/addons/fluentbit/fluentbit.yaml +++ b/addons/fluentbit/fluentbit.yaml @@ -177,7 +177,7 @@ spec: Time_Key @ts Logstash_Format On Logstash_Prefix kubernetes_audit - Retry_Limit False + Retry_Limit 10 Buffer_Size 512KB [OUTPUT] Name es @@ -188,7 +188,7 @@ spec: Time_Key @ts Logstash_Format On Logstash_Prefix kubernetes_cluster - Retry_Limit False + Retry_Limit 10 Buffer_Size 512KB [OUTPUT] Name es @@ -199,7 +199,7 @@ spec: Time_Key @ts Logstash_Format On Logstash_Prefix kubernetes_host - Retry_Limit False + Retry_Limit 10 Buffer_Size 512KB [OUTPUT] Name es @@ -210,7 +210,7 @@ spec: Time_Key @ts Logstash_Format On Logstash_Prefix kubernetes_host_kernel - Retry_Limit False + Retry_Limit 10 Buffer_Size 512KB ## https://docs.fluentbit.io/manual/pipeline/parsers From 695747ca715ab407024fecbec5ad5433346600f1 Mon Sep 17 00:00:00 2001 From: alejandroEsc Date: Fri, 9 Oct 2020 13:38:44 -0700 Subject: [PATCH 2/3] revision bump --- addons/fluentbit/fluentbit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/fluentbit/fluentbit.yaml b/addons/fluentbit/fluentbit.yaml index 5b025546..990c2901 100644 --- a/addons/fluentbit/fluentbit.yaml +++ b/addons/fluentbit/fluentbit.yaml @@ -6,7 +6,7 @@ metadata: labels: kubeaddons.mesosphere.io/name: fluentbit annotations: - catalog.kubeaddons.mesosphere.io/addon-revision: "1.5.6-2" + catalog.kubeaddons.mesosphere.io/addon-revision: "1.5.6-3" appversion.kubeaddons.mesosphere.io/fluentbit: "1.5.6" values.chart.helm.kubeaddons.mesosphere.io/fluentbit: "https://raw.githubusercontent.com/fluent/helm-charts/5e20fbc/charts/fluent-bit/values.yaml" # the older versions were being deployed from stable/fluent-bit From 9af888845e61f395a5fa3fed036421128633f35d Mon Sep 17 00:00:00 2001 From: Alejandro Escobar Date: Fri, 9 Oct 2020 15:26:34 -0700 Subject: [PATCH 3/3] Update addons/fluentbit/fluentbit.yaml Co-authored-by: Branden Rolston --- addons/fluentbit/fluentbit.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/addons/fluentbit/fluentbit.yaml b/addons/fluentbit/fluentbit.yaml index 990c2901..3eec5a14 100644 --- a/addons/fluentbit/fluentbit.yaml +++ b/addons/fluentbit/fluentbit.yaml @@ -177,7 +177,10 @@ spec: Time_Key @ts Logstash_Format On Logstash_Prefix kubernetes_audit - Retry_Limit 10 + # A small fraction of audit logs will be rejected by Elasticsearch because values may have an inconsistent type. + # We set a low retry limit here in order to prevent fluent-bit from backing up while it repeatedly retries flushing these logs. + # See https://github.com/uken/fluent-plugin-elasticsearch#random-400---rejected-by-elasticsearch-is-occured-why. + Retry_Limit 1 Buffer_Size 512KB [OUTPUT] Name es