fix: fluentbit should not keep retrying indefinitely

addons/fluentbit/fluentbit.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     kubeaddons.mesosphere.io/name: fluentbit
   annotations:
-    catalog.kubeaddons.mesosphere.io/addon-revision: "1.5.6-2"
+    catalog.kubeaddons.mesosphere.io/addon-revision: "1.5.6-3"
     appversion.kubeaddons.mesosphere.io/fluentbit: "1.5.6"
     values.chart.helm.kubeaddons.mesosphere.io/fluentbit: "https://raw.githubusercontent.com/fluent/helm-charts/5e20fbc/charts/fluent-bit/values.yaml"
     # the older versions were being deployed from stable/fluent-bit
@@ -177,7 +177,10 @@ spec:
               Time_Key @ts
               Logstash_Format On
               Logstash_Prefix kubernetes_audit
-              Retry_Limit False
+              # A small fraction of audit logs will be rejected by Elasticsearch because values may have an inconsistent type.
+              # We set a low retry limit here in order to prevent fluent-bit from backing up while it repeatedly retries flushing these logs.
+              # See https://github.com/uken/fluent-plugin-elasticsearch#random-400---rejected-by-elasticsearch-is-occured-why.
+              Retry_Limit 1
               Buffer_Size 512KB
           [OUTPUT]
               Name es
@@ -188,7 +191,7 @@ spec:
               Time_Key @ts
               Logstash_Format On
               Logstash_Prefix kubernetes_cluster
-              Retry_Limit False
+              Retry_Limit 10
               Buffer_Size 512KB
           [OUTPUT]
               Name es
@@ -199,7 +202,7 @@ spec:
               Time_Key @ts
               Logstash_Format On
               Logstash_Prefix kubernetes_host
-              Retry_Limit False
+              Retry_Limit 10
               Buffer_Size 512KB
           [OUTPUT]
               Name es
@@ -210,7 +213,7 @@ spec:
               Time_Key @ts
               Logstash_Format On
               Logstash_Prefix kubernetes_host_kernel
-              Retry_Limit False
+              Retry_Limit 10
               Buffer_Size 512KB
 
         ## https://docs.fluentbit.io/manual/pipeline/parsers