From 94677fe336e5c7e16271256fc073be410a6b6de4 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 25 May 2022 14:50:19 -0700 Subject: [PATCH 1/3] Add memberlist.libsonnet and modify loki services so we can set the port when using memberlist for the ring. Signed-off-by: Callum Styan --- .../ksonnet/loki/boltdb_shipper.libsonnet | 9 +- production/ksonnet/loki/distributor.libsonnet | 4 +- .../ksonnet/loki/index-gateway.libsonnet | 11 +- production/ksonnet/loki/ingester.libsonnet | 4 +- production/ksonnet/loki/memberlist.libsonnet | 144 ++++++++++++++++++ .../ksonnet/loki/query-scheduler.libsonnet | 4 +- production/ksonnet/loki/ruler.libsonnet | 4 +- 7 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 production/ksonnet/loki/memberlist.libsonnet diff --git a/production/ksonnet/loki/boltdb_shipper.libsonnet b/production/ksonnet/loki/boltdb_shipper.libsonnet index 31f67f748a2bc..b3ad5b4a5a9d8 100644 --- a/production/ksonnet/loki/boltdb_shipper.libsonnet +++ b/production/ksonnet/loki/boltdb_shipper.libsonnet @@ -49,14 +49,13 @@ target: 'compactor', } else {}, - local compactor_ports = - [ - containerPort.new(name='http-metrics', port=$._config.http_listen_port), - ], + compactor_ports: [ + containerPort.new(name='http-metrics', port=$._config.http_listen_port), + ], compactor_container:: if $._config.using_boltdb_shipper then container.new('compactor', $._images.compactor) + - container.withPorts(compactor_ports) + + container.withPorts($.compactor_ports) + container.withArgsMixin(k.util.mapToFlags($.compactor_args)) + container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) + container.mixin.readinessProbe.httpGet.withPath('/ready') + diff --git a/production/ksonnet/loki/distributor.libsonnet b/production/ksonnet/loki/distributor.libsonnet index d9f3b668d64d2..510a6a81e4ed6 100644 --- a/production/ksonnet/loki/distributor.libsonnet +++ b/production/ksonnet/loki/distributor.libsonnet @@ -8,9 +8,11 @@ local k = import 'ksonnet-util/kausal.libsonnet'; target: 'distributor', }, + distributor_ports:: $.util.defaultPorts, + distributor_container:: container.new('distributor', $._images.distributor) + - container.withPorts($.util.defaultPorts) + + container.withPorts($.distributor_ports) + container.withArgsMixin(k.util.mapToFlags($.distributor_args)) + container.mixin.readinessProbe.httpGet.withPath('/ready') + container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) + diff --git a/production/ksonnet/loki/index-gateway.libsonnet b/production/ksonnet/loki/index-gateway.libsonnet index 66bcaf6fb85a0..9acec7efa3316 100644 --- a/production/ksonnet/loki/index-gateway.libsonnet +++ b/production/ksonnet/loki/index-gateway.libsonnet @@ -34,15 +34,14 @@ target: 'index-gateway', } else {}, - local index_gateway_ports = - [ - containerPort.new(name='http-metrics', port=$._config.http_listen_port), - containerPort.new(name='grpc', port=9095), - ], + index_gateway_ports: [ + containerPort.new(name='http-metrics', port=$._config.http_listen_port), + containerPort.new(name='grpc', port=9095), + ], index_gateway_container:: if $._config.use_index_gateway then container.new('index-gateway', $._images.index_gateway) + - container.withPorts(index_gateway_ports) + + container.withPorts($.index_gateway_ports) + container.withArgsMixin(k.util.mapToFlags($.index_gateway_args)) + container.withVolumeMountsMixin([volumeMount.new('index-gateway-data', '/data')]) + container.mixin.readinessProbe.httpGet.withPath('/ready') + diff --git a/production/ksonnet/loki/ingester.libsonnet b/production/ksonnet/loki/ingester.libsonnet index e24b064058e78..826a083a64574 100644 --- a/production/ksonnet/loki/ingester.libsonnet +++ b/production/ksonnet/loki/ingester.libsonnet @@ -15,9 +15,11 @@ local k = import 'ksonnet-util/kausal.libsonnet'; 'ingester.max-transfer-retries': 0, } else {}, + ingester_ports: $.util.defaultPorts, + ingester_container:: container.new('ingester', $._images.ingester) + - container.withPorts($.util.defaultPorts) + + container.withPorts($.ingester_ports) + container.withArgsMixin(k.util.mapToFlags($.ingester_args)) + container.mixin.readinessProbe.httpGet.withPath('/ready') + container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) + diff --git a/production/ksonnet/loki/memberlist.libsonnet b/production/ksonnet/loki/memberlist.libsonnet new file mode 100644 index 0000000000000..63ca458c1b7cc --- /dev/null +++ b/production/ksonnet/loki/memberlist.libsonnet @@ -0,0 +1,144 @@ +{ + local memberlistConfig = { + 'memberlist.abort-if-join-fails': false, + 'memberlist.bind-port': gossipRingPort, + 'memberlist.join': 'gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort], + }, + + local setupGossipRing(storeOption, consulHostnameOption, multiStoreOptionsPrefix) = if $._config.multikv_migration_enabled then { + [storeOption]: 'multi', + [multiStoreOptionsPrefix + '.primary']: $._config.multikv_primary, + [multiStoreOptionsPrefix + '.secondary']: $._config.multikv_secondary, + // don't remove consul.hostname, it may still be needed. + } else { + [storeOption]: 'memberlist', + [consulHostnameOption]: null, + }, + + _config+:: { + gossip_member_label: 'loki_gossip_member', + // Enables use of memberlist for all rings, instead of consul. If multikv_migration_enabled is true, consul hostname is still configured, + // but "primary" KV depends on value of multikv_primary. + memberlist_ring_enabled: false, + + // Migrating from consul to memberlist is a multi-step process: + // 1) Enable multikv_migration_enabled, with primary=consul, secondary=memberlist, and multikv_mirror_enabled=false, restart components. + // 2) Set multikv_mirror_enabled=true. This doesn't require restart. + // 3) Swap multikv_primary and multikv_secondary, ie. multikv_primary=memberlist, multikv_secondary=consul. This doesn't require restart. + // 4) Set multikv_migration_enabled=false and multikv_migration_teardown=true. This requires a restart, but components will now use only memberlist. + // 5) Set multikv_migration_teardown=false. This doesn't require a restart. + multikv_migration_enabled: false, + multikv_migration_teardown: false, + multikv_primary: 'consul', + multikv_secondary: 'memberlist', + multikv_switch_primary_secondary: false, + multikv_mirror_enabled: false, + + multi_kv_obj: if $._config.multikv_migration_enabled then { + store: 'multi', + multi: { + primary: $._config.multikv_primary, + secondary: $._config.multikv_secondary, + }, + } else { + store: 'memberlist', + consul: { + host: null, + }, + }, + + loki+: { + ingester+: { + lifecycler+: { + ring+: { + kvstore+: $._config.multi_kv_obj, + }, + }, + }, + + distributor+: { + ring+: { + kvstore+: $._config.multi_kv_obj, + }, + }, + + ruler+: if !$._config.ruler_enabled then {} else { + ring+: { + kvstore: $._config.multi_kv_obj, + }, + }, + + // other components pick this up from common + common+: { + ring+: { + kvstore+: $._config.multi_kv_obj + }, + }, + + memberlist: { + abort_if_cluster_join_fails: false, + + # Expose this port on all distributor, ingester + # and querier replicas. + bind_port: gossipRingPort, + + # You can use a headless k8s service for all distributor, + # ingester and querier components. + join_members: ['gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort],], + + max_join_backoff: '1m', + max_join_retries: '10', + min_join_backoff: '1s', + }, + + }, + + // When doing migration via multi KV store, this section can be used + // to configure runtime parameters of multi KV store + multi_kv_config: if !$._config.multikv_migration_enabled && !$._config.multikv_migration_teardown then {} else { + primary: if $._config.multikv_switch_primary_secondary then $._config.multikv_secondary else $._config.multikv_primary, + mirror_enabled: $._config.multikv_mirror_enabled, + }, + }, + + local gossipRingPort = 7946, + + local containerPort = $.core.v1.containerPort, + local gossipPort = containerPort.newNamed(name='gossip-ring', containerPort=gossipRingPort), + + + + compactor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + distributor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + index_gateway_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + ingester_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + query_scheduler_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + ruler_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], + + // Don't add label to matcher, only to pod labels. + local gossipLabel = $.apps.v1.statefulSet.spec.template.metadata.withLabelsMixin({ [$._config.gossip_member_label]: 'true' }), + + compactor_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel, + distributor_deployment+: if !$._config.memberlist_ring_enabled then {} else gossipLabel, + index_gateway_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel, + ingester_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel, + query_scheduler_deployment+: if !$._config.memberlist_ring_enabled then {} else gossipLabel, + ruler_deployment+: if !$._config.memberlist_ring_enabled || !$._config.ruler_enabled then {} else gossipLabel, + + // Headless service (= no assigned IP, DNS returns all targets instead) pointing to gossip network members. + gossip_ring_service: + if !$._config.memberlist_ring_enabled then null + else + local service = $.core.v1.service; + local servicePort = $.core.v1.servicePort; + + local ports = [ + servicePort.newNamed('gossip-ring', gossipRingPort, gossipRingPort) + + servicePort.withProtocol('TCP'), + ]; + service.new( + 'gossip-ring', // name + { [$._config.gossip_member_label]: 'true' }, // point to all gossip members + ports, + ) + service.mixin.spec.withClusterIp('None'), // headless service +} diff --git a/production/ksonnet/loki/query-scheduler.libsonnet b/production/ksonnet/loki/query-scheduler.libsonnet index 31d76c3e9441f..bb63b08c12172 100644 --- a/production/ksonnet/loki/query-scheduler.libsonnet +++ b/production/ksonnet/loki/query-scheduler.libsonnet @@ -32,10 +32,12 @@ local k = import 'ksonnet-util/kausal.libsonnet'; } else {}, + query_scheduler_ports:: $.util.grpclbDefaultPorts, + local container = k.core.v1.container, query_scheduler_container:: if $._config.query_scheduler_enabled then container.new('query-scheduler', $._images.query_scheduler) + - container.withPorts($.util.grpclbDefaultPorts) + + container.withPorts($.query_scheduler_ports) + container.withArgsMixin(k.util.mapToFlags($.query_scheduler_args)) + $.jaeger_mixin + k.util.resourcesRequests('2', '600Mi') + diff --git a/production/ksonnet/loki/ruler.libsonnet b/production/ksonnet/loki/ruler.libsonnet index 14ac4d3829ad9..a9874b2bca091 100644 --- a/production/ksonnet/loki/ruler.libsonnet +++ b/production/ksonnet/loki/ruler.libsonnet @@ -19,10 +19,12 @@ local k = import 'ksonnet-util/kausal.libsonnet'; stateful_rulers: if self.using_boltdb_shipper && !self.use_index_gateway then true else super.stateful_rulers, }, + ruler_ports:: $.util.defaultPorts, + ruler_container:: if $._config.ruler_enabled then container.new('ruler', $._images.ruler) + - container.withPorts($.util.defaultPorts) + + container.withPorts($.ruler_ports) + container.withArgsMixin(k.util.mapToFlags($.ruler_args)) + k.util.resourcesRequests('1', '6Gi') + k.util.resourcesLimits('16', '16Gi') + From 114138303c2f80f2d7ba4eba34618bfce96f48b0 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 26 May 2022 10:20:30 -0700 Subject: [PATCH 2/3] Missed the + for the ruler ring config here. Signed-off-by: Callum Styan --- production/ksonnet/loki/memberlist.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/ksonnet/loki/memberlist.libsonnet b/production/ksonnet/loki/memberlist.libsonnet index 63ca458c1b7cc..bce9164f7ca24 100644 --- a/production/ksonnet/loki/memberlist.libsonnet +++ b/production/ksonnet/loki/memberlist.libsonnet @@ -64,7 +64,7 @@ ruler+: if !$._config.ruler_enabled then {} else { ring+: { - kvstore: $._config.multi_kv_obj, + kvstore+: $._config.multi_kv_obj, }, }, From 150ec4b41f66ca2a3cb92720fc1cb5b8670d2ebf Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 26 May 2022 10:35:52 -0700 Subject: [PATCH 3/3] Fix linting. Signed-off-by: Callum Styan --- production/ksonnet/loki/memberlist.libsonnet | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/production/ksonnet/loki/memberlist.libsonnet b/production/ksonnet/loki/memberlist.libsonnet index bce9164f7ca24..5868dae8f7b8b 100644 --- a/production/ksonnet/loki/memberlist.libsonnet +++ b/production/ksonnet/loki/memberlist.libsonnet @@ -71,26 +71,25 @@ // other components pick this up from common common+: { ring+: { - kvstore+: $._config.multi_kv_obj + kvstore+: $._config.multi_kv_obj, }, }, memberlist: { abort_if_cluster_join_fails: false, - # Expose this port on all distributor, ingester - # and querier replicas. + // Expose this port on all distributor, ingester + // and querier replicas. bind_port: gossipRingPort, - # You can use a headless k8s service for all distributor, - # ingester and querier components. - join_members: ['gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort],], + // You can use a headless k8s service for all distributor, + // ingester and querier components. + join_members: ['gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort]], max_join_backoff: '1m', max_join_retries: '10', min_join_backoff: '1s', }, - }, // When doing migration via multi KV store, this section can be used @@ -106,8 +105,6 @@ local containerPort = $.core.v1.containerPort, local gossipPort = containerPort.newNamed(name='gossip-ring', containerPort=gossipRingPort), - - compactor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], distributor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort], index_gateway_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],