diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go index 446ef4138..df42504c5 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go @@ -104,6 +104,8 @@ type Tail struct { PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"` //Skips empty lines in the log file from any further processing or output. SkipEmptyLines *bool `json:"skipEmptyLines,omitempty"` + // Threaded mechanism allows input plugin to run in a separate thread which helps to desaturate the main pipeline. + Threaded *string `json:"threaded,omitempty"` } func (_ *Tail) Name() string { @@ -202,5 +204,8 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) { if t.SkipEmptyLines != nil { kvs.Insert("Skip_Empty_Lines", fmt.Sprint(*t.SkipEmptyLines)) } + if t.Threaded != nil { + kvs.Insert("Threaded", *t.Threaded) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go b/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go index 713fa5624..f8c2b4c45 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go @@ -44,6 +44,10 @@ type Kafka struct { //The default value is 10 times, the interval between each retry is 1 second. //Setting the queue_full_retries value to 0 set's an unlimited number of retries. QueueFullRetries *int64 `json:"queueFullRetries,omitempty"` + // Limit the maximum number of Chunks in the filesystem for the current output logical destination. + TotalLimitSize string `json:"totalLimitSize,omitempty"` + // Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. + Workers *int32 `json:"workers,omitempty"` } func (*Kafka) Name() string { @@ -88,5 +92,12 @@ func (k *Kafka) Params(_ plugins.SecretLoader) (*params.KVs, error) { return fmt.Sprintf("rdkafka.%s", k), v }) + if k.TotalLimitSize != "" { + kvs.Insert("storage.total_limit_size", k.TotalLimitSize) + } + if k.Workers != nil { + kvs.Insert("workers", fmt.Sprint(*k.Workers)) + } + return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/loki_types.go b/apis/fluentbit/v1alpha2/plugins/output/loki_types.go index 99ab2e973..cd069e9cb 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/loki_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/loki_types.go @@ -59,6 +59,10 @@ type Loki struct { *plugins.TLS `json:"tls,omitempty"` // Include fluentbit networking options for this output-plugin *plugins.Networking `json:"networking,omitempty"` + // Limit the maximum number of Chunks in the filesystem for the current output logical destination. + TotalLimitSize string `json:"totalLimitSize,omitempty"` + // Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. + Workers *int32 `json:"workers,omitempty"` } // implement Section() method @@ -144,5 +148,11 @@ func (l *Loki) Params(sl plugins.SecretLoader) (*params.KVs, error) { } kvs.Merge(net) } + if l.TotalLimitSize != "" { + kvs.Insert("storage.total_limit_size", l.TotalLimitSize) + } + if l.Workers != nil { + kvs.Insert("workers", fmt.Sprint(*l.Workers)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go index 3519a6b86..2183ee05e 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go @@ -93,7 +93,7 @@ type OpenSearch struct { // When enabled, mapping types is removed and Type option is ignored. Types are deprecated in APIs in v7.0. This options is for v7.0 or later. SuppressTypeName *bool `json:"suppressTypeName,omitempty"` // Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. - Workers *int32 `json:"Workers,omitempty"` + Workers *int32 `json:"workers,omitempty"` *plugins.TLS `json:"tls,omitempty"` // Include fluentbit networking options for this output-plugin *plugins.Networking `json:"networking,omitempty"` diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index f3ce7b5c1..485212be1 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -840,6 +840,10 @@ spec: tagRegex: description: Set a regex to exctract fields from the file type: string + threaded: + description: Threaded mechanism allows input plugin to run in + a separate thread which helps to desaturate the main pipeline. + type: string type: object tcp: description: TCP defines the TCP input plugin configuration diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index 54e6eef18..a8ad2998a 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -2070,6 +2070,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -2474,10 +2484,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -2497,12 +2517,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -2835,6 +2849,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index c992e204f..d99653d39 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -2070,6 +2070,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -2474,10 +2484,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -2497,12 +2517,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -2835,6 +2849,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string diff --git a/charts/fluent-operator/templates/fluentbit-output-loki.yaml b/charts/fluent-operator/templates/fluentbit-output-loki.yaml index 5deddbc23..b6134179c 100644 --- a/charts/fluent-operator/templates/fluentbit-output-loki.yaml +++ b/charts/fluent-operator/templates/fluentbit-output-loki.yaml @@ -37,6 +37,7 @@ metadata: fluentbit.fluent.io/component: logging spec: matchRegex: (?:kube|service)\.(.*) + retry_limit: "{{ .retryLimit }}" loki: {{ if .host }}host: {{ .host | quote }}{{ end }} {{ if .port }}port: {{ .port }}{{ end }} diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index 82bd8bcf0..7c5ca5292 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -290,6 +290,7 @@ fluentbit: loki: # Switch for generation of fluentbit loki ClusterOutput (and loki basic auth http user and pass secrets if required) enable: false # Bool + retryLimit: "no_limits" host: 127.0.0.1 # String port: 3100 # Int # Either, give http{User,Password},tenantID string values specifying them directly diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index f3ce7b5c1..485212be1 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -840,6 +840,10 @@ spec: tagRegex: description: Set a regex to exctract fields from the file type: string + threaded: + description: Threaded mechanism allows input plugin to run in + a separate thread which helps to desaturate the main pipeline. + type: string type: object tcp: description: TCP defines the TCP input plugin configuration diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 54e6eef18..a8ad2998a 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -2070,6 +2070,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -2474,10 +2484,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -2497,12 +2517,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -2835,6 +2849,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index c992e204f..d99653d39 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -2070,6 +2070,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -2474,10 +2484,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -2497,12 +2517,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -2835,6 +2849,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 74771bd5e..abc3f8d3c 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -2956,6 +2956,10 @@ spec: tagRegex: description: Set a regex to exctract fields from the file type: string + threaded: + description: Threaded mechanism allows input plugin to run in + a separate thread which helps to desaturate the main pipeline. + type: string type: object tcp: description: TCP defines the TCP input plugin configuration @@ -6115,6 +6119,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -6519,10 +6533,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -6542,12 +6566,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -6880,6 +6898,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string @@ -34855,6 +34879,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -35259,10 +35293,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -35282,12 +35326,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -35620,6 +35658,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index c12ae2949..2b85aa1e6 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -2956,6 +2956,10 @@ spec: tagRegex: description: Set a regex to exctract fields from the file type: string + threaded: + description: Threaded mechanism allows input plugin to run in + a separate thread which helps to desaturate the main pipeline. + type: string type: object tcp: description: TCP defines the TCP input plugin configuration @@ -6115,6 +6119,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -6519,10 +6533,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -6542,12 +6566,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -6880,6 +6898,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string @@ -34855,6 +34879,16 @@ spec: If only one topic is set, that one will be used for all records. Instead if multiple topics exists, the one set in the record by Topic_Key will be used. type: string + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer type: object kinesis: description: Kinesis defines Kinesis Output configuration. @@ -35259,10 +35293,20 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string uri: description: Specify a custom HTTP URI. It must start with forward slash. type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer required: - host type: object @@ -35282,12 +35326,6 @@ spec: opensearch: description: OpenSearch defines OpenSearch Output configuration. properties: - Workers: - description: Enables dedicated thread(s) for this output. Default - value is set since version 1.8.13. For previous versions is - 0. - format: int32 - type: integer awsAuth: description: Enable AWS Sigv4 Authentication for Amazon OpenSearch Service. @@ -35620,6 +35658,12 @@ spec: type: description: Type name type: string + workers: + description: Enables dedicated thread(s) for this output. Default + value is set since version 1.8.13. For previous versions is + 0. + format: int32 + type: integer writeOperation: description: Operation to use to write in bulk requests. type: string