Replace PodAntiAffinity with TopologySpreadConstraints

Fixes #8168 Signed-off-by: Takumi Sue <[email protected]>
linkerd · Jul 7, 2022 · a7a6c3b · a7a6c3b
1 parent 73e4bf7
commit a7a6c3b
Show file tree

Hide file tree

Showing 18 changed files with 44 additions and 41 deletions.
diff --git a/charts/linkerd-control-plane/README.md b/charts/linkerd-control-plane/README.md
@@ -151,9 +151,9 @@ Kubernetes: `>=1.21.0-0`
 | enableEndpointSlices | bool | `true` | enables the use of EndpointSlice informers for the destination service; enableEndpointSlices should be set to true only if EndpointSlice K8s feature gate is on |
 | enableH2Upgrade | bool | `true` | Allow proxies to perform transparent HTTP/2 upgrading |
 | enablePSP | bool | `false` | Add a PSP resource and bind it to the control plane ServiceAccounts. Note PSP has been deprecated since k8s v1.21 |
-| enablePodAntiAffinity | bool | `false` | enables pod anti affinity creation on deployments for high availability |
 | enablePodDisruptionBudget | bool | `false` | enables the creation of pod disruption budgets for control plane components |
 | enablePprof | bool | `false` | enables the use of pprof endpoints on control plane component's admin servers |
+| enableTopologySpreadConstraints | bool | `false` | enables TopologySpreadConstraints creation on deployments for high availability |
 | identity.externalCA | bool | `false` | If the linkerd-identity-trust-roots ConfigMap has already been created |
 | identity.issuer.clockSkewAllowance | string | `"20s"` | Amount of time to allow for clock skew within a Linkerd cluster |
 | identity.issuer.issuanceLifetime | string | `"24h0m0s"` | Amount of time for which the Identity issuer should certify identity |

diff --git a/charts/linkerd-control-plane/templates/destination.yaml b/charts/linkerd-control-plane/templates/destination.yaml
@@ -168,6 +168,7 @@ spec:
       {{- include "linkerd.node-selector" . | nindent 6 }}
       {{- $_ := set $tree "component" "destination" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       {{- if not (empty .Values.destinationProxyResources) }}
       {{- $r := merge .Values.destinationProxyResources .Values.proxy.resources }}

diff --git a/charts/linkerd-control-plane/templates/identity.yaml b/charts/linkerd-control-plane/templates/identity.yaml
@@ -141,6 +141,7 @@ spec:
       {{- include "linkerd.node-selector" . | nindent 6 }}
       {{- $_ := set $tree "component" "identity" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       - args:
         - identity

diff --git a/charts/linkerd-control-plane/templates/proxy-injector.yaml b/charts/linkerd-control-plane/templates/proxy-injector.yaml
@@ -54,6 +54,7 @@ spec:
       {{- include "linkerd.node-selector" . | nindent 6 }}
       {{- $_ := set $tree "component" "proxy-injector" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       {{- if not (empty .Values.proxyInjectorProxyResources) }}
       {{- $r := merge .Values.proxyInjectorProxyResources .Values.proxy.resources }}

diff --git a/charts/linkerd-control-plane/values-ha.yaml b/charts/linkerd-control-plane/values-ha.yaml
@@ -11,8 +11,8 @@ deploymentStrategy:
     maxUnavailable: 1
     maxSurge: 25%
 
-# -- add PodAntiAffinity to each control plane workload
-enablePodAntiAffinity: true
+# -- add TopologySpreadConstraints to each control plane workload
+enableTopologySpreadConstraints: true
 
 # nodeAffinity: 
 

diff --git a/charts/linkerd-control-plane/values.yaml b/charts/linkerd-control-plane/values.yaml
@@ -32,8 +32,8 @@ deploymentStrategy:
 # enableEndpointSlices should be set to true only if EndpointSlice K8s feature
 # gate is on
 enableEndpointSlices: true
-# -- enables pod anti affinity creation on deployments for high availability
-enablePodAntiAffinity: false
+# -- enables TopologySpreadConstraints creation on deployments for high availability
+enableTopologySpreadConstraints: false
 # -- enables the use of pprof endpoints on control plane component's admin
 # servers
 enablePprof: false

diff --git a/charts/partials/templates/_affinity.tpl b/charts/partials/templates/_affinity.tpl
@@ -1,38 +1,11 @@
-{{ define "linkerd.pod-affinity" -}}
-podAntiAffinity:
-  preferredDuringSchedulingIgnoredDuringExecution:
-  - podAffinityTerm:
-      labelSelector:
-        matchExpressions:
-        - key: {{ default "linkerd.io/control-plane-component" .label }}
-          operator: In
-          values:
-          - {{ .component }}
-      topologyKey: failure-domain.beta.kubernetes.io/zone
-    weight: 100
-  requiredDuringSchedulingIgnoredDuringExecution:
-  - labelSelector:
-      matchExpressions:
-      - key: {{ default "linkerd.io/control-plane-component" .label }}
-        operator: In
-        values:
-        - {{ .component }}
-    topologyKey: kubernetes.io/hostname
-{{- end }}
-
 {{ define "linkerd.node-affinity" -}}
 nodeAffinity:
 {{- toYaml .Values.nodeAffinity | trim | nindent 2 }}
 {{- end }}
 
 {{ define "linkerd.affinity" -}}
-{{- if or .Values.enablePodAntiAffinity .Values.nodeAffinity -}}
-affinity:
-{{- end }}
-{{- if .Values.enablePodAntiAffinity -}}
-{{- include "linkerd.pod-affinity" . | nindent 2 }}
-{{- end }}
 {{- if .Values.nodeAffinity -}}
+affinity:
 {{- include "linkerd.node-affinity" . | nindent 2 }}
 {{- end }}
 {{- end }}
diff --git a/charts/partials/templates/_topologyspreadconstraints.tpl b/charts/partials/templates/_topologyspreadconstraints.tpl
@@ -0,0 +1,23 @@
+{{ define "linkerd.topologySpreadConstraints" -}}
+{{- if .Values.enableTopologySpreadConstraints }}
+topologySpreadConstraints:
+- maxSkew: 1
+  topologyKey: failure-domain.beta.kubernetes.io/zone
+  whenUnsatisfiable: ScheduleAnyway
+  labelSelector:
+    matchExpressions:
+    - key: {{ default "linkerd.io/control-plane-component" .label }}
+      operator: In
+      values:
+      - {{ .component }}
+- maxSkew: 1
+  topologyKey: kubernetes.io/hostname
+  whenUnsatisfiable: DoNotSchedule
+  labelSelector:
+    matchExpressions:
+    - key: {{ default "linkerd.io/control-plane-component" .label }}
+      operator: In
+      values:
+      - {{ .component }}
+{{- end }}
+{{- end }}
diff --git a/multicluster/charts/linkerd-multicluster/README.md b/multicluster/charts/linkerd-multicluster/README.md
@@ -72,7 +72,7 @@ Kubernetes: `>=1.21.0-0`
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | enablePSP | bool | `false` | Create Roles and RoleBindings to associate this extension's ServiceAccounts to the control plane PSP resource. This requires that `enabledPSP` is set to true on the control plane install. Note PSP has been deprecated since k8s v1.21 |
-| enablePodAntiAffinity | bool | `false` | Enables Pod Anti Affinity logic to balance the placement of replicas across hosts and zones for High Availability. Enable this only when you have multiple replicas of components. |
+| enableTopologySpreadConstraints | bool | `false` | Enables Topology Spread Constraints logic to balance the placement of replicas across hosts and zones for High Availability. Enable this only when you have multiple replicas of components. |
 | gateway.UID | int | `2103` | User id under which the gateway shall be ran |
 | gateway.enabled | bool | `true` | If the gateway component should be installed |
 | gateway.loadBalancerIP | string | `""` | Set loadBalancerIP on gateway service |

diff --git a/multicluster/charts/linkerd-multicluster/templates/gateway.yaml b/multicluster/charts/linkerd-multicluster/templates/gateway.yaml
@@ -38,6 +38,7 @@ spec:
       {{- $_ := set $tree "component" .Values.gateway.name -}}
       {{- $_ := set $tree "label" "app" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
         - name: pause
           image: gcr.io/google_containers/pause:3.2

diff --git a/multicluster/charts/linkerd-multicluster/values-ha.yaml b/multicluster/charts/linkerd-multicluster/values-ha.yaml
@@ -1,6 +1,6 @@
 gateway:
   replicas: 3
 
-enablePodAntiAffinity: true
+enableTopologySpreadConstraints: true
 
 # nodeAffinity:
diff --git a/multicluster/charts/linkerd-multicluster/values.yaml b/multicluster/charts/linkerd-multicluster/values.yaml
@@ -49,10 +49,10 @@ identityTrustDomain: cluster.local
 # deprecated since k8s v1.21
 enablePSP: false
 
-# -- Enables Pod Anti Affinity logic to balance the placement of replicas
+# -- Enables Topology Spread Constraints logic to balance the placement of replicas
 # across hosts and zones for High Availability.
 # Enable this only when you have multiple replicas of components.
-enablePodAntiAffinity: false
+enableTopologySpreadConstraints: false
 
 # -- NodeAffinity section, See the
 # [K8S documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)

diff --git a/viz/charts/linkerd-viz/README.md b/viz/charts/linkerd-viz/README.md
@@ -95,7 +95,7 @@ Kubernetes: `>=1.21.0-0`
 | defaultRegistry | string | `"cr.l5d.io/linkerd"` | Docker registry for all viz components |
 | defaultUID | int | `2103` | UID for all the viz components |
 | enablePSP | bool | `false` | NodeAffinity section, See the [K8S documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for more information nodeAffinity: -- Create Roles and RoleBindings to associate this extension's ServiceAccounts to the control plane PSP resource. This requires that `enabledPSP` is set to true on the control plane install. Note PSP has been deprecated since k8s v1.21 |
-| enablePodAntiAffinity | bool | `false` | Enables Pod Anti Affinity logic to balance the placement of replicas across hosts and zones for High Availability. Enable this only when you have multiple replicas of components. |
+| enableTopologySpreadConstraints | bool | `false` | Enables Topology Spread Constraints logic to balance the placement of replicas across hosts and zones for High Availability. Enable this only when you have multiple replicas of components. |
 | grafana.externalUrl | string | `nil` | url of a Grafana instance hosted off-cluster. Cannot be set if grafana.url is set. The reverse proxy will not be used for this URL. |
 | grafana.uidPrefix | string | `nil` | prefix for Grafana dashboard UID's, used when grafana.externalUrl is set. |
 | grafana.url | string | `nil` | url of an in-cluster Grafana instance with reverse proxy configured, used by the Linkerd viz web dashboard to provide direct links to specific Grafana dashboards. Cannot be set if grafana.externalUrl is set. See the [Linkerd documentation](https://linkerd.io/2/tasks/grafana) for more information |

diff --git a/viz/charts/linkerd-viz/templates/metrics-api.yaml b/viz/charts/linkerd-viz/templates/metrics-api.yaml
@@ -69,6 +69,7 @@ spec:
       {{- $_ := set $tree "component" "metrics-api" -}}
       {{- $_ := set $tree "label" "component" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       - args:
         - -controller-namespace={{.Values.linkerdNamespace}}

diff --git a/viz/charts/linkerd-viz/templates/tap-injector.yaml b/viz/charts/linkerd-viz/templates/tap-injector.yaml
@@ -72,6 +72,7 @@ spec:
       {{- $_ := set $tree "component" "tap-injector" -}}
       {{- $_ := set $tree "label" "component" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       - args:
         - injector

diff --git a/viz/charts/linkerd-viz/templates/tap.yaml b/viz/charts/linkerd-viz/templates/tap.yaml
@@ -81,6 +81,7 @@ spec:
       {{- $_ := set $tree "component" "tap" -}}
       {{- $_ := set $tree "label" "component" -}}
       {{- include "linkerd.affinity" $tree | nindent 6 }}
+      {{- include "linkerd.topologySpreadConstraints" $tree | nindent 6 }}
       containers:
       - args:
         - api

diff --git a/viz/charts/linkerd-viz/values-ha.yaml b/viz/charts/linkerd-viz/values-ha.yaml
@@ -2,7 +2,7 @@
 # Usage:
 #   helm install -f values.yaml -f values-ha.yaml
 
-enablePodAntiAffinity: true
+enableTopologySpreadConstraints: true
 
 # nodeAffinity:
 

diff --git a/viz/charts/linkerd-viz/values.yaml b/viz/charts/linkerd-viz/values.yaml
@@ -41,10 +41,10 @@ imagePullSecrets: []
 # for more information
 tolerations: &default_tolerations
 
-# -- Enables Pod Anti Affinity logic to balance the placement of replicas
+# -- Enables Topology Spread Constraints logic to balance the placement of replicas
 # across hosts and zones for High Availability.
 # Enable this only when you have multiple replicas of components.
-enablePodAntiAffinity: false
+enableTopologySpreadConstraints: false
 
 # -- NodeAffinity section, See the
 # [K8S documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity)