diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet index 03fddf5e533..5b5b3f0700b 100644 --- a/operations/mimir/config.libsonnet +++ b/operations/mimir/config.libsonnet @@ -58,6 +58,8 @@ // Use the Cortex chunks storage engine by default, while giving the ability // to switch to tsdb storage. storage_engine: 'chunks', + // Secondary storage engine is only used for querying. + querier_second_storage_engine: null, storage_tsdb_bucket_name: error 'must specify GCS bucket name to store TSDB blocks', store_gateway_replication_factor: 3, @@ -113,7 +115,7 @@ storeConfig: self.storeMemcachedChunksConfig, - storeMemcachedChunksConfig: if $._config.memcached_chunks_enabled && $._config.storage_engine == 'chunks' then + storeMemcachedChunksConfig: if $._config.memcached_chunks_enabled && ($._config.storage_engine == 'chunks' || $._config.querier_second_storage_engine == 'chunks') then { 'store.chunks-cache.memcached.hostname': 'memcached.%s.svc.cluster.local' % $._config.namespace, 'store.chunks-cache.memcached.service': 'memcached-client', @@ -131,8 +133,8 @@ // TSDB blocks storage configuration, used only when 'tsdb' storage // engine is explicitly enabled. storageTSDBConfig: ( - if $._config.storage_engine != 'tsdb' then {} else { - 'store.engine': 'tsdb', + if $._config.storage_engine == 'tsdb' || $._config.querier_second_storage_engine == 'tsdb' then { + 'store.engine': $._config.storage_engine, // May still be chunks 'experimental.tsdb.dir': '/data/tsdb', 'experimental.tsdb.bucket-store.sync-dir': '/data/tsdb', 'experimental.tsdb.bucket-store.ignore-deletion-marks-delay': '1h', @@ -147,7 +149,7 @@ 'experimental.store-gateway.sharding-ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, 'experimental.store-gateway.sharding-ring.prefix': '', 'experimental.store-gateway.replication-factor': $._config.store_gateway_replication_factor, - } + } else {} ), // Shared between the Ruler and Querier diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet index 2fa7b5ced6c..165fc9829fe 100644 --- a/operations/mimir/ingester.libsonnet +++ b/operations/mimir/ingester.libsonnet @@ -75,10 +75,12 @@ local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, - ingester_pdb: + newIngesterPdb(pdbName, ingesterName):: podDisruptionBudget.new() + - podDisruptionBudget.mixin.metadata.withName('ingester-pdb') + - podDisruptionBudget.mixin.metadata.withLabels({ name: 'ingester-pdb' }) + - podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: name }) + + podDisruptionBudget.mixin.metadata.withName(pdbName) + + podDisruptionBudget.mixin.metadata.withLabels({ name: pdbName }) + + podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: ingesterName }) + podDisruptionBudget.mixin.spec.withMaxUnavailable(1), + + ingester_pdb: self.newIngesterPdb('ingester-pdb', name), } diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet index 8dd4bc7b88e..044fd8ce175 100644 --- a/operations/mimir/querier.libsonnet +++ b/operations/mimir/querier.libsonnet @@ -10,7 +10,7 @@ { target: 'querier', - // Increase HTTP server response write timeout, as we were seeing some + // Increase HTTP server response write timeout, as we were seeing some // queries that return a lot of data timeing out. 'server.http-write-timeout': '1m', @@ -22,6 +22,8 @@ 'querier.frontend-address': 'query-frontend-discovery.%(namespace)s.svc.cluster.local:9095' % $._config, 'querier.frontend-client.grpc-max-send-msg-size': 100 << 20, + 'querier.second-store-engine': $._config.querier_second_storage_engine, + 'log.level': 'debug', }, diff --git a/operations/mimir/tsdb.libsonnet b/operations/mimir/tsdb.libsonnet index fe2da963608..5b13850170b 100644 --- a/operations/mimir/tsdb.libsonnet +++ b/operations/mimir/tsdb.libsonnet @@ -84,18 +84,17 @@ 'ingester.tokens-file-path': '/data/tokens', }, - ingester_container+:: - container.withVolumeMountsMixin([ - volumeMount.new('ingester-data', '/data'), - ]), - - ingester_statefulset: - statefulSet.new('ingester', 3, [$.ingester_container], ingester_data_pvc) + - statefulSet.mixin.spec.withServiceName('ingester') + + newIngesterStatefulSet(name, container):: + statefulSet.new(name, 3, [ + container + $.core.v1.container.withVolumeMountsMixin([ + volumeMount.new('ingester-data', '/data'), + ]), + ], ingester_data_pvc) + + statefulSet.mixin.spec.withServiceName(name) + statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: 'ingester' }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: 'ingester' } + $.ingester_deployment_labels) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: 'ingester' }) + + statefulSet.mixin.metadata.withLabels({ name: name }) + + statefulSet.mixin.spec.template.metadata.withLabels({ name: name } + $.ingester_deployment_labels) + + statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(600) + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + @@ -108,6 +107,8 @@ // ready). statefulSet.mixin.spec.withPodManagementPolicy('Parallel'), + ingester_statefulset: self.newIngesterStatefulSet('ingester', $.ingester_container), + ingester_service: $.util.serviceFor($.ingester_statefulset, $.ingester_service_ignored_labels),