Merge pull request #420 from grafana/backport-improvements

Upstream some config improvements
grafana · Nov 22, 2021 · aaa283e · aaa283e
2 parents b2bd8a4 + 63f0dd0
commit aaa283e
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,10 @@
 * [CHANGE] Split `cortex_api` recording rule group into three groups. This is a workaround for large clusters where this group can become slow to evaluate. #401
 * [CHANGE] Increased `CortexIngesterReachingSeriesLimit` warning threshold from 70% to 80% and critical threshold from 85% to 90%. #404
 * [CHANGE] Rename ruler_s3_bucket_name and ruler_gcs_bucket_name to ruler_storage_bucket_name: #415
+* [CHANGE] Fine-tuned rolling update policy for distributor, querier, query-frontend, query-scheduler. #420
+* [CHANGE] Increased memcached metadata/chunks/index-queries max connections from 4k to 16k. #420
+* [CHANGE] Disabled step alignment in query-frontend to be compliant with PromQL. #420
+* [CHANGE] Do not limit compactor CPU and request a number of cores equal to the configured concurrency. #420
 * [ENHANCEMENT] Add overrides config to compactor. This allows setting retention configs per user. #386
 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317
 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. #328

diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet
@@ -57,7 +57,9 @@
   distributor_deployment:
     deployment.new('distributor', 3, [$.distributor_container], $.distributor_deployment_labels) +
     (if $._config.cortex_distributor_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
-    $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex'),
+    $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   local service = $.core.v1.service,
 

diff --git a/cortex/memcached.libsonnet b/cortex/memcached.libsonnet
@@ -34,6 +34,7 @@ memcached {
     $.memcached {
       name: 'memcached-index-queries',
       max_item_size: '%dm' % [$._config.memcached_index_queries_max_item_size_mb],
+      connection_limit: 16384,
     }
   else {},
 
@@ -54,7 +55,7 @@ memcached {
       // Save memory by more tightly provisioning memcached chunks.
       memory_limit_mb: 6 * 1024,
       overprovision_factor: 1.05,
-      connection_limit: 4096,
+      connection_limit: 16384,
 
       local container = $.core.v1.container,
     }
@@ -65,7 +66,7 @@ memcached {
     $.memcached {
       name: 'memcached-metadata',
       max_item_size: '%dm' % [$._config.memcached_metadata_max_item_size_mb],
-      connection_limit: 4096,
+      connection_limit: 16384,
 
       // Metadata cache doesn't need much memory.
       memory_limit_mb: 512,

diff --git a/cortex/querier.libsonnet b/cortex/querier.libsonnet
@@ -58,6 +58,8 @@
     deployment.new(name, $._config.querier.replicas, [container], $.querier_deployment_labels) +
     (if $._config.cortex_querier_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
     $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) +
     $.storage_config_mixin,
 
   querier_deployment:

diff --git a/cortex/query-frontend.libsonnet b/cortex/query-frontend.libsonnet
@@ -17,7 +17,7 @@
       'querier.split-queries-by-interval': '24h',
 
       // Cache query results.
-      'querier.align-querier-with-step': true,
+      'querier.align-querier-with-step': false,
       'querier.cache-results': true,
       'frontend.memcached.hostname': 'memcached-frontend.%s.svc.cluster.local' % $._config.namespace,
       'frontend.memcached.service': 'memcached-client',
@@ -52,7 +52,9 @@
   newQueryFrontendDeployment(name, container)::
     deployment.new(name, $._config.queryFrontend.replicas, [container]) +
     $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
-    (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity),
+    (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(1) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   query_frontend_deployment: self.newQueryFrontendDeployment('query-frontend', $.query_frontend_container),
 

diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet
@@ -25,7 +25,10 @@
   newQuerySchedulerDeployment(name, container)::
     deployment.new(name, 2, [container]) +
     $.util.configVolumeMount('overrides', '/etc/cortex') +
-    $.util.antiAffinity,
+    $.util.antiAffinity +
+    // Do not run more query-schedulers than expected.
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else
     self.newQuerySchedulerDeployment('query-scheduler', $.query_scheduler_container),

diff --git a/cortex/tsdb.libsonnet b/cortex/tsdb.libsonnet
@@ -172,8 +172,9 @@
     container.withPorts($.compactor_ports) +
     container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
-    $.util.resourcesRequests('1', '6Gi') +
-    $.util.resourcesLimits($._config.cortex_compactor_max_concurrency, '6Gi') +
+    // Do not limit compactor CPU and request enough cores to honor configured max concurrency.
+    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') +
+    $.util.resourcesLimits(null, '6Gi') +
     $.util.readinessProbe +
     $.jaeger_mixin,