Skip to content

Commit

Permalink
Merge pull request #219 from joejulian/helm-charts-racks
Browse files Browse the repository at this point in the history
redpanda: add rack awareness feature
  • Loading branch information
joejulian authored Dec 6, 2022
2 parents 6065963 + bc6d23a commit c2145b5
Show file tree
Hide file tree
Showing 11 changed files with 271 additions and 44 deletions.
8 changes: 8 additions & 0 deletions .github/annotate_kind_nodes.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -xeuo pipefail

KIND_CLUSTER_NAME="$1"
kubectl annotate "node/${KIND_CLUSTER_NAME}-worker" topology.kubernetes.io/zone=rack1
kubectl annotate "node/${KIND_CLUSTER_NAME}-worker2" topology.kubernetes.io/zone=rack2
kubectl annotate "node/${KIND_CLUSTER_NAME}-worker3" topology.kubernetes.io/zone=rack3
kubectl annotate "node/${KIND_CLUSTER_NAME}-worker4" topology.kubernetes.io/zone=rack4
4 changes: 4 additions & 0 deletions .github/workflows/pull_requests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ jobs:
run: kubectl get nodes
if: steps.list-changed.outputs.changed == 'true'

- name: Annotate nodes for rack awareness
run: .github/annotate_kind_nodes.sh chart-testing
if: steps.list-changed.outputs.changed == 'true'

- name: install cert-manager
run: kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.0/cert-manager.yaml
if: steps.list-changed.outputs.changed == 'true'
Expand Down
2 changes: 1 addition & 1 deletion charts/redpanda/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type: application
# The chart version and the app version are not the same and will not track
# together. The chart version is a semver representation of changes to this
# chart.
version: 2.3.12
version: 2.3.13
# The app version is the default version of Redpanda to install.
appVersion: v22.3.3
# kubeVersion must be suffixed with "-0" to be able to match cloud providers
Expand Down
4 changes: 1 addition & 3 deletions charts/redpanda/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ See the [60-Second Guide for Kubernetes][kubernetes-qs-dev]
If you have improvements that can be made to this Helm chart, please consider becoming a contributor.
See our [Contributing][contributing] document for more details.

[values]: https://github.com/redpanda-data/helm-charts/blob/main/redpanda/values.yaml
[values]: https://github.com/redpanda-data/helm-charts/blob/main/charts/redpanda/values.yaml
[examples]: https://github.com/redpanda-data/helm-charts/blob/main/examples/README.md
[contributing]: https://github.com/redpanda-data/helm-charts/blob/main/CONTRIBUTING.md
[kubernetes-qs-dev]: https://docs.redpanda.com/docs/quickstart/kubernetes-qs-dev/


21 changes: 21 additions & 0 deletions charts/redpanda/ci/06-rack-awareness.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
rackAwareness:
enabled: true
rbac:
enabled: true
serviceaccount:
create: true
3 changes: 3 additions & 0 deletions charts/redpanda/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ data:
redpanda:
{{- if (include "redpanda-atleast-22-3-0" . | fromJson).bool }}
empty_seed_starts_cluster: false
{{- if .Values.rackAwareness.enabled }}
enable_rack_awareness: true
{{- end }}
{{- end }}
{{- if not (include "redpanda-atleast-22-1-1" . | fromJson).bool }}
enable_sasl: {{ dig "sasl" "enabled" false .Values.auth }}
Expand Down
70 changes: 70 additions & 0 deletions charts/redpanda/templates/rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
---
{{- if .Values.rbac.enabled -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "redpanda.fullname" . }}
labels:
helm.sh/chart: {{ template "redpanda.chart" . }}
app.kubernetes.io/name: {{ template "redpanda.name" . }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/component: {{ template "redpanda.name" . }}
{{- with .Values.commonLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "redpanda.fullname" . }}
labels:
helm.sh/chart: {{ template "redpanda.chart" . }}
app.kubernetes.io/name: {{ template "redpanda.name" . }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/component: {{ template "redpanda.name" . }}
{{- with .Values.commonLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ include "redpanda.fullname" . }}
subjects:
- kind: ServiceAccount
name: {{ include "redpanda.serviceAccountName" . }}
namespace: {{ .Release.Namespace | quote }}
{{- end }}
99 changes: 61 additions & 38 deletions charts/redpanda/templates/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ limitations under the License.
*/}}

{{- $values := .Values }}
{{- $advertiseAddress := include "redpanda.kafka.internal.advertise.address" . -}}
{{- $advertiseAddress := include "redpanda.kafka.internal.advertise.address" . }}
---
apiVersion: apps/v1
kind: StatefulSet
Expand Down Expand Up @@ -59,6 +59,7 @@ spec:
spec:
securityContext:
{{- toYaml .Values.statefulset.podSecurityContext | nindent 8 }}
serviceAccountName: {{ include "redpanda.serviceAccountName" . }}
initContainers:
- name: set-datadir-ownership
image: busybox:latest
Expand All @@ -76,46 +77,68 @@ spec:
fieldPath: metadata.name
- name: EXTERNAL_ADDRESSES
value: {{ .Values.external.addresses }}
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
args:
- >
CONFIG=/etc/redpanda/redpanda.yaml;
cp /tmp/base-config/redpanda.yaml "$CONFIG";
NODE_ID=${SERVICE_NAME##*-};
- |
set -xe
CONFIG=/etc/redpanda/redpanda.yaml
POD_ORDINAL=${SERVICE_NAME##*-}
# Setup config files
cp /tmp/base-config/redpanda.yaml "${CONFIG}"
{{- if (include "redpanda-atleast-22-1-1" . | fromJson).bool }}
cp /tmp/base-config/bootstrap.yaml /etc/redpanda/.bootstrap.yaml;
cp /tmp/base-config/bootstrap.yaml /etc/redpanda/.bootstrap.yaml
{{- end }}
{{- if not (include "redpanda-atleast-22-3-0" . | fromJson).bool }}
rpk --config "$CONFIG" redpanda config set redpanda.node_id $NODE_ID;
if [ "$NODE_ID" = "0" ]; then
rpk --config "$CONFIG" redpanda config set redpanda.seed_servers '[]' --format yaml;
fi;
# Configure bootstrap
## Not used for Redpanda v22.3.0+
rpk --config "${CONFIG}" redpanda config set redpanda.node_id "${POD_ORDINAL}"
if [ "${POD_ORDINAL}" = "0" ]; then
rpk --config "${CONFIG}" redpanda config set redpanda.seed_servers '[]' --format yaml
fi
{{- end }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].name internal ;
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].address {{ $advertiseAddress }} ;
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].port {{ .Values.listeners.kafka.port }} ;
{{- $listenerIndex := 1 -}}
{{- range $name, $listener := .Values.listeners.kafka.external -}}
{{- $enabled := dig "enabled" $values.external.enabled $listener -}}
{{- $listenerNodePortEnabled := and $enabled (eq (dig "type" $values.external.type $listener) "NodePort") -}}
{{- $advertiseKafkaHost := $advertiseAddress -}}
{{- $advertiseKafkaPort := $listener.nodePort -}}
{{- if $listenerNodePortEnabled -}}
{{- if $values.external.addresses -}}
NODE_INDEX=`expr $NODE_ID + 1`;
NODE_ADDRESS=`echo $EXTERNAL_ADDRESSES | cut -d ' ' -f $NODE_INDEX`;
{{- if eq $values.external.addressType "ip" -}}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address $NODE_ADDRESS;
{{- else -}}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address $NODE_ADDRESS.{{ $values.external.domain }};
{{- end -}}
{{- else -}}
{{- $advertiseKafkaHost = printf "$(SERVICE_NAME).%s" $values.external.domain -}}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address {{ $advertiseKafkaHost }} ;
{{- end -}}
{{- end -}}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].name {{ $name }} ;
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].port {{ $advertiseKafkaPort }} ;
{{- $listenerIndex = add $listenerIndex 1 -}}
# Configure internal kafka listeners
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].name internal
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].address {{ $advertiseAddress }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[0].port {{ .Values.listeners.kafka.port }}
# Configure external kafka listeners
{{- $listenerIndex := 1 }}
{{- range $name, $listener := .Values.listeners.kafka.external }}
{{- $enabled := dig "enabled" $values.external.enabled $listener }}
{{- $listenerNodePortEnabled := and $enabled (eq (dig "type" $values.external.type $listener) "NodePort") }}
{{- $advertiseKafkaHost := $advertiseAddress }}
{{- $advertiseKafkaPort := $listener.nodePort }}
{{- if $listenerNodePortEnabled }}
{{- if $values.external.addresses }}
NODE_INDEX=`expr $NODE_ID + 1`
NODE_ADDRESS=`echo $EXTERNAL_ADDRESSES | cut -d ' ' -f $NODE_INDEX`
{{- if eq $values.external.addressType "ip" }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address $NODE_ADDRESS
{{- else }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address $NODE_ADDRESS.{{ $values.external.domain }}
{{- end }}
{{- else }}
{{- $advertiseKafkaHost = printf "$(SERVICE_NAME).%s" $values.external.domain }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].address {{ $advertiseKafkaHost }}
{{- end }}
{{- end }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].name {{ $name }}
rpk redpanda --config "$CONFIG" config set redpanda.advertised_kafka_api[{{ $listenerIndex }}].port {{ $advertiseKafkaPort }}
{{- $listenerIndex = add $listenerIndex 1 }}
{{- end }}
{{- if (include "redpanda-atleast-22-3-0" . | fromJson).bool }}
{{- if .Values.rackAwareness.enabled }}
# Configure Rack Awareness
RACK=$(curl --silent --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --fail -H 'Authorization: Bearer '$(cat /run/secrets/kubernetes.io/serviceaccount/token) "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}/api/v1/nodes/${KUBERNETES_NODE_NAME}?pretty=true" | grep {{ .Values.rackAwareness.nodeAnnotation | quote | squote }} | sed 's/.*": "\([^"]\+\).*/\1/')
rpk --config "$CONFIG" redpanda config set redpanda.rack "${RACK}"
{{- end }}
{{- end }}
volumeMounts:
- name: {{ template "redpanda.fullname" . }}
Expand Down Expand Up @@ -232,7 +255,7 @@ spec:
- --advertise-rpc-addr={{ $advertiseAddress }}:{{ .Values.listeners.rpc.port }}
- --advertise-pandaproxy-addr=internal://{{ $advertiseAddress }}:{{ .Values.listeners.http.port }},
{{- range $name, $listener := .Values.listeners.http.external -}}
{{ $name}}://{{ $advertiseAddress }}:{{ $listener.nodePort }},
{{ $name }}://{{ $advertiseAddress }}:{{ $listener.nodePort }},
{{- end }}
ports:
{{- range $name, $listener := .Values.listeners }}
Expand Down Expand Up @@ -297,7 +320,7 @@ spec:
{{- end }}
secretName: {{ template "redpanda.fullname" $ }}-{{ $name }}-cert
{{- end }}
{{- end -}}
{{- end }}
{{- if or .Values.statefulset.nodeAffinity .Values.statefulset.podAffinity .Values.statefulset.podAntiAffinity }}
affinity:
{{- with .Values.statefulset.nodeAffinity }}
Expand Down
55 changes: 55 additions & 0 deletions charts/redpanda/templates/tests/test-rack-awareness.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.rackAwareness.enabled -}}
{{- if not (or (include "tls-enabled" . | fromJson).bool (include "sasl-enabled" .)) -}}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "redpanda.fullname" . }}-test-rack-awareness
namespace: {{ .Release.Namespace | quote }}
labels:
helm.sh/chart: {{ template "redpanda.chart" . }}
app.kubernetes.io/name: {{ template "redpanda.name" . }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/component: {{ template "redpanda.name" . }}
{{- with .Values.commonLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": test
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
spec:
backoffLimit: 2
completions: 1
parallelism: 1
ttlSecondsAfterFinished: 120
template:
spec:
restartPolicy: Never
containers:
- name: {{ template "redpanda.name" . }}
image: {{ .Values.image.repository}}:{{ template "redpanda.tag" . }}
command:
- /bin/bash
- -c
- |
set -e
curl --silent --fail http://{{ include "redpanda.fullname" . }}:{{ .Values.listeners.admin.port }}/v1/node_config | grep '"rack":"rack[1-4]"'
rpk redpanda admin config print --host {{ include "redpanda.fullname" . }}:{{ .Values.listeners.admin.port }} | grep '"enable_rack_awareness": true'
{{- end -}}
{{- end -}}
30 changes: 30 additions & 0 deletions charts/redpanda/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,21 @@
}
}
},
"rackAwareness": {
"type": "object",
"required": [
"enabled",
"nodeAnnotation"
],
"properties": {
"enabled": {
"type": "boolean"
},
"nodeAnnotation": {
"type": "string"
}
}
},
"auth": {
"type": "object",
"required": [
Expand Down Expand Up @@ -577,6 +592,21 @@
}
}
},
"rbac": {
"type": "object",
"required": [
"enabled",
"annotations"
],
"properties": {
"enabled": {
"type": "boolean"
},
"annotations": {
"type": "object"
}
}
},
"tuning": {
"type": "object",
"properties": {
Expand Down
Loading

0 comments on commit c2145b5

Please sign in to comment.