From 8357877ca56ba489e9e4426963692713d52a2a1b Mon Sep 17 00:00:00 2001 From: Connor Catlett Date: Wed, 16 Aug 2023 15:29:22 +0000 Subject: [PATCH] Additional DaemonSets feature Signed-off-by: Connor Catlett --- .../templates/_node-windows.tpl | 199 +++++++++++++++ charts/aws-ebs-csi-driver/templates/_node.tpl | 224 +++++++++++++++++ .../templates/node-windows.yaml | 209 +-------------- charts/aws-ebs-csi-driver/templates/node.yaml | 237 ++---------------- charts/aws-ebs-csi-driver/values.yaml | 9 + deploy/kubernetes/base/node.yaml | 1 - docs/additional-daemonsets.md | 44 ++++ 7 files changed, 503 insertions(+), 420 deletions(-) create mode 100644 charts/aws-ebs-csi-driver/templates/_node-windows.tpl create mode 100644 charts/aws-ebs-csi-driver/templates/_node.tpl create mode 100644 docs/additional-daemonsets.md diff --git a/charts/aws-ebs-csi-driver/templates/_node-windows.tpl b/charts/aws-ebs-csi-driver/templates/_node-windows.tpl new file mode 100644 index 0000000000..90b9a8591b --- /dev/null +++ b/charts/aws-ebs-csi-driver/templates/_node-windows.tpl @@ -0,0 +1,199 @@ +{{- define "node-windows" }} +{{- if .Values.node.enableWindows }} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{ toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + windowsOptions: + runAsUserName: "ContainerAdministrator" + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + value: unix:/csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + periodSeconds: 90 + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + - name: probe-dir + mountPath: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=unix:/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + - name: probe-dir + emptyDir: {} +{{- end }} +{{- end }} diff --git a/charts/aws-ebs-csi-driver/templates/_node.tpl b/charts/aws-ebs-csi-driver/templates/_node.tpl new file mode 100644 index 0000000000..bd2224b844 --- /dev/null +++ b/charts/aws-ebs-csi-driver/templates/_node.tpl @@ -0,0 +1,224 @@ +{{- define "node" }} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + - key: "ebs.csi.aws.com/agent-not-ready" + operator: "Exists" + {{- end }} + {{- with .Values.node.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: {{ .Values.node.kubeletPath }} + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.node.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ printf "%s/plugins/ebs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- range .Values.sidecars.nodeDriverRegistrar.additionalArgs }} + - {{ . }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + periodSeconds: 90 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: probe-dir + mountPath: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.additionalArgs }} + - {{ . }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: probe-dir + emptyDir: {} + {{- with .Values.node.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/aws-ebs-csi-driver/templates/node-windows.yaml index 4baa2832ac..c11e04bbd8 100644 --- a/charts/aws-ebs-csi-driver/templates/node-windows.yaml +++ b/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -1,197 +1,14 @@ -{{- if .Values.node.enableWindows }} -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node-windows - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{ toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: windows - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - --v={{ .Values.node.logLevel }} - {{- if .Values.node.otelTracing }} - - --enable-otel-tracing=true - {{- end}} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.otelTracing }} - - name: OTEL_SERVICE_NAME - value: {{ .otelServiceName }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .otelExporterEndpoint }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: C:\var\lib\kubelet - mountPropagation: "None" - - name: plugin-dir - mountPath: C:\csi - - name: csi-proxy-disk-pipe - mountPath: \\.\pipe\csi-proxy-disk-v1 - - name: csi-proxy-volume-pipe - mountPath: \\.\pipe\csi-proxy-volume-v1 - - name: csi-proxy-filesystem-pipe - mountPath: \\.\pipe\csi-proxy-filesystem-v1 - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - securityContext: - windowsOptions: - runAsUserName: "ContainerAdministrator" - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: unix:/csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar.exe - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - periodSeconds: 90 - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - - name: registration-dir - mountPath: C:\registration - - name: probe-dir - mountPath: C:\var\lib\kubelet\plugins\ebs.csi.aws.com - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=unix:/csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: C:\var\lib\kubelet - type: Directory - - name: plugin-dir - hostPath: - path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: C:\var\lib\kubelet\plugins_registry - type: Directory - - name: csi-proxy-disk-pipe - hostPath: - path: \\.\pipe\csi-proxy-disk-v1 - type: "" - - name: csi-proxy-volume-pipe - hostPath: - path: \\.\pipe\csi-proxy-volume-v1 - type: "" - - name: csi-proxy-filesystem-pipe - hostPath: - path: \\.\pipe\csi-proxy-filesystem-v1 - type: "" - - name: probe-dir - emptyDir: {} +{{$defaultArgs := dict + "NodeName" "ebs-csi-node-windows" +}} +{{- include "node" (deepCopy $ | mustMerge $defaultArgs) -}} + +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-windows-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} {{- end }} diff --git a/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/aws-ebs-csi-driver/templates/node.yaml index 511d31a557..9f36688aee 100644 --- a/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/aws-ebs-csi-driver/templates/node.yaml @@ -1,223 +1,14 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{- toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - - key: "ebs.csi.aws.com/agent-not-ready" - operator: "Exists" - {{- end }} - {{- with .Values.node.securityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - --v={{ .Values.node.logLevel }} - {{- if .Values.node.otelTracing }} - - --enable-otel-tracing=true - {{- end}} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.otelTracing }} - - name: OTEL_SERVICE_NAME - value: {{ .otelServiceName }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .otelExporterEndpoint }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - {{- with .Values.controller.envFrom }} - envFrom: - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: {{ .Values.node.kubeletPath }} - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: device-dir - mountPath: /dev - {{- with .Values.node.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.node.containerSecurityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: {{ printf "%s/plugins/ebs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - {{- range .Values.sidecars.nodeDriverRegistrar.additionalArgs }} - - {{ . }} - {{- end }} - {{- with .Values.controller.envFrom }} - envFrom: - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - periodSeconds: 90 - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: probe-dir - mountPath: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=/csi/csi.sock - {{- range .Values.sidecars.livenessProbe.additionalArgs }} - - {{ . }} - {{- end }} - {{- with .Values.controller.envFrom }} - envFrom: - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.livenessProbe.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: {{ .Values.node.kubeletPath }} - type: Directory - - name: plugin-dir - hostPath: - path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory - - name: probe-dir - emptyDir: {} - {{- with .Values.node.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} +{{$defaultArgs := dict + "NodeName" "ebs-csi-node" +}} +{{- include "node" (deepCopy $ | mustMerge $defaultArgs) -}} + +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} diff --git a/charts/aws-ebs-csi-driver/values.yaml b/charts/aws-ebs-csi-driver/values.yaml index 3d0aeaf581..5a6ff4f32f 100644 --- a/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/aws-ebs-csi-driver/values.yaml @@ -371,6 +371,15 @@ node: # otelServiceName: ebs-csi-node # otelExporterEndpoint: "http://localhost:4317" +additionalDaemonSets: + # Additional node DaemonSets, using the node config structure + # See docs/additional-daemonsets.md for more information + # + # example: + # nodeSelector: + # node.kubernetes.io/instance-type: c5.large + # volumeAttachLimit: 15 + storageClasses: [] # Add StorageClass resources like: # - name: ebs-sc diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index e25a724c06..12498314fd 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -1,6 +1,5 @@ --- # Source: aws-ebs-csi-driver/templates/node.yaml -# Node Service kind: DaemonSet apiVersion: apps/v1 metadata: diff --git a/docs/additional-daemonsets.md b/docs/additional-daemonsets.md new file mode 100644 index 0000000000..b63653b754 --- /dev/null +++ b/docs/additional-daemonsets.md @@ -0,0 +1,44 @@ +# Additional Node DaemonSets Feature + +In some situations, it is desirable to create multiple node `DaemonSet`s of the EBS CSI Driver. For example, when specifying `.node.volumeAttachLimit`, the limit may differ by node instance type or role. + +The EBS CSI Driver Helm chart supports the creation of additional `DaemonSet`s via the `.additionalDaemonSets` parameter. Node configuration from the values supplied to `.node` are taken as a default, with the values supplied in the `.additionalDaemonSets` configuration as overrides. An additional Linux (and Windows, if enabled) `DaemonSet` will be rendered for each entry in `additionalDaemonSets`. + +**WARNING: The EBS CSI Driver does not support running multiple node pods on the same node. If you use this feature, ensure that all nodes are targeted by no more than one `DaemonSet`s.** + +## Example + +For example, the following configuration would produce three `DaemonSet`s: + +```yaml +node: + nodeSelector: + node.kubernetes.io/instance-type: c5.large + volumeAttachLimit: 25 + resources: + limits: + memory: 512Mi + +additionalNodeDaemonSets: + big: + nodeSelector: + node.kubernetes.io/instance-type: m7i.48xlarge + volumeAttachLimit: 100 + small: + nodeSelector: + node.kubernetes.io/instance-type: t3.medium + volumeAttachLimit: 5 + resources: + limits: + memory: 128Mi +``` + +The `DaemonSet`s would be configured as follows: + +- `ebs-csi-node` (the default `DaemonSet`) +Runs on `c5.large` instances with a volume limit of 25 and 512Mi memory limit. +- `ebs-csi-node-big` +Runs on `m7i.48xlarge` instances with a volume limit of 100 and 512Mi memory limit. +Note how the volume limit is inherited from the `.node` configuration because this config does not specify them. This way, `.node` can be used to set defaults for all the `DaemonSet`s. +- `ebs-csi-node-small` +Runs on `t3.medium` instances with a volume limit of 5 and 128Mi memory limit.