From b9690e2eb8fb9015b3f34882a1fa4e7267638559 Mon Sep 17 00:00:00 2001 From: Jatin Suri <52750297+jatinsu@users.noreply.github.com> Date: Wed, 3 Jul 2024 01:37:25 -0400 Subject: [PATCH] feat(operator): Add support for the volume API (#13369) --- operator/CHANGELOG.md | 4 +++ operator/apis/loki/v1/lokistack_types.go | 7 +++++ .../loki-operator.clusterserviceversion.yaml | 12 ++++++++ .../loki.grafana.com_lokistacks.yaml | 10 +++++++ .../bases/loki.grafana.com_lokistacks.yaml | 10 +++++++ .../loki-operator.clusterserviceversion.yaml | 12 ++++++++ operator/docs/operator/api.md | 12 ++++++++ .../manifests/internal/config/build_test.go | 30 +++++++++++++++++++ .../internal/config/loki-config.yaml | 2 ++ .../internal/config/loki-runtime-config.yaml | 3 ++ operator/internal/manifests/internal/sizes.go | 3 ++ 11 files changed, 105 insertions(+) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 5ab99c69f60bf..bbe1c75c7315a 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,9 @@ ## Main +## Release 5.6.21 + +- [13369](https://github.com/grafana/loki/pull/13369) **jatinsu**: feat(operator): Add support for the volume API + ## Release 5.6.20 - [13066](https://github.com/grafana/loki/pull/13066) **xperimental**: Use a minimum value for replay memory ceiling diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 6531681bb805f..be7fbce8cc85b 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -562,6 +562,13 @@ type QueryLimitSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Cardinality Limit" CardinalityLimit int32 `json:"cardinalityLimit,omitempty"` + + // MaxVolumeSeries defines the maximum number of aggregated series in a log-volume response + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Volume Series" + MaxVolumeSeries int32 `json:"maxVolumeSeries,omitempty"` } // IngestionLimitSpec defines the limits applied at the ingestion path. diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index bad1ce8957af1..291d8c8e9ac2e 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -390,6 +390,12 @@ spec: path: limits.global.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: MaxVolumeSeries defines the maximum number of aggregated series + in a log-volume response + displayName: Max Volume Series + path: limits.global.queries.maxVolumeSeries + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Timeout when querying ingesters or storage during the execution of a query request. displayName: Query Timeout @@ -474,6 +480,12 @@ spec: path: limits.tenants.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: MaxVolumeSeries defines the maximum number of aggregated series + in a log-volume response + displayName: Max Volume Series + path: limits.tenants.queries.maxVolumeSeries + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Timeout when querying ingesters or storage during the execution of a query request. displayName: Query Timeout diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml index e2b67f8ac1c4c..8b844392707dd 100644 --- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml @@ -170,6 +170,11 @@ spec: unique series that is returned by a metric query. format: int32 type: integer + maxVolumeSeries: + description: MaxVolumeSeries defines the maximum number + of aggregated series in a log-volume response + format: int32 + type: integer queryTimeout: default: 3m description: Timeout when querying ingesters or storage @@ -298,6 +303,11 @@ spec: of unique series that is returned by a metric query. format: int32 type: integer + maxVolumeSeries: + description: MaxVolumeSeries defines the maximum number + of aggregated series in a log-volume response + format: int32 + type: integer queryTimeout: default: 3m description: Timeout when querying ingesters or storage diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 530ba681b8f18..87fef6a8bd519 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -153,6 +153,11 @@ spec: unique series that is returned by a metric query. format: int32 type: integer + maxVolumeSeries: + description: MaxVolumeSeries defines the maximum number + of aggregated series in a log-volume response + format: int32 + type: integer queryTimeout: default: 3m description: Timeout when querying ingesters or storage @@ -281,6 +286,11 @@ spec: of unique series that is returned by a metric query. format: int32 type: integer + maxVolumeSeries: + description: MaxVolumeSeries defines the maximum number + of aggregated series in a log-volume response + format: int32 + type: integer queryTimeout: default: 3m description: Timeout when querying ingesters or storage diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml index df85cabbd150c..9d73cf6e5f967 100644 --- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -245,6 +245,12 @@ spec: path: limits.global.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: MaxVolumeSeries defines the maximum number of aggregated series + in a log-volume response + displayName: Max Volume Series + path: limits.global.queries.maxVolumeSeries + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Timeout when querying ingesters or storage during the execution of a query request. displayName: Query Timeout @@ -329,6 +335,12 @@ spec: path: limits.tenants.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: MaxVolumeSeries defines the maximum number of aggregated series + in a log-volume response + displayName: Max Volume Series + path: limits.tenants.queries.maxVolumeSeries + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Timeout when querying ingesters or storage during the execution of a query request. displayName: Query Timeout diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index b32ea2c7a9f3c..da3e4ed9caf18 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1862,6 +1862,18 @@ int32

CardinalityLimit defines the cardinality limit for index queries.

+ + +maxVolumeSeries
+ +int32 + + + +(Optional) +

MaxVolumeSeries defines the maximum number of aggregated series in a log-volume response

+ + diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 74db8c520a2c5..061df2f38ee7d 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -109,6 +109,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -197,6 +199,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -359,6 +362,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -452,6 +457,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, Tenants: map[string]lokiv1.LimitsTemplateSpec{ @@ -701,6 +707,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -843,6 +851,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -1052,6 +1061,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -1194,6 +1205,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -1404,6 +1416,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -1559,6 +1573,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -1794,6 +1809,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -1892,6 +1909,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, Retention: &lokiv1.RetentionLimitSpec{ Days: 15, @@ -2089,6 +2107,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 2m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -2257,6 +2277,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "2m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -2522,6 +2543,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_port: 7946 @@ -2647,6 +2670,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -2841,6 +2865,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_addr: ${HASH_RING_INSTANCE_ADDR} @@ -2930,6 +2956,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -3095,6 +3122,8 @@ limits_config: per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 memberlist: abort_if_cluster_join_fails: true advertise_addr: ${HASH_RING_INSTANCE_ADDR} @@ -3184,6 +3213,7 @@ overrides: MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 9316c55905d0c..8e0e4c6bf67ee 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -163,6 +163,8 @@ limits_config: cardinality_limit: {{ .Stack.Limits.Global.QueryLimits.CardinalityLimit }} max_streams_matchers_per_query: 1000 query_timeout: {{ .Stack.Limits.Global.QueryLimits.QueryTimeout }} + volume_enabled: true + volume_max_series: {{ .Stack.Limits.Global.QueryLimits.MaxVolumeSeries }} {{- if .Retention.Enabled }}{{- with .Stack.Limits.Global.Retention }} retention_period: {{.Days}}d {{- with .Streams }} diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index 7f011fcc3ab63..ddbd109a3bb60 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -47,6 +47,9 @@ overrides: {{- if $spec.QueryLimits.CardinalityLimit }} cardinality_limit: {{ $spec.QueryLimits.CardinalityLimit }} {{- end }} + {{- if $spec.QueryLimits.MaxVolumeSeries }} + max_volume_series: {{ $spec.QueryLimits.MaxVolumeSeries }} + {{- end }} {{- end -}} {{- with $spec.Retention }} retention_period: {{ .Days }}d diff --git a/operator/internal/manifests/internal/sizes.go b/operator/internal/manifests/internal/sizes.go index 30a12be6b718e..3308238e3a1b5 100644 --- a/operator/internal/manifests/internal/sizes.go +++ b/operator/internal/manifests/internal/sizes.go @@ -228,6 +228,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -284,6 +285,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, }, @@ -340,6 +342,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxQuerySeries: 500, QueryTimeout: "1m", CardinalityLimit: 100000, + MaxVolumeSeries: 1000, }, }, },