Manage podAntiAffinity for dex

CHANGELOG.md

@@ -12,6 +12,11 @@
   hostname, so that if it's possible each `CoreDNS` pods will sit on different infra node
   (PR[#3579](https://github.com/scality/metalk8s/pull/3579))
 
+- Allow to manage soft and hard `podAntiAffinity` for `Dex` from Cluster
+  and Services Configurations, with a default soft anti-affinity on hostname,
+  so that if it's possible each `Dex` pods will sit on a different infra node
+  (PR[#3614](https://github.com/scality/metalk8s/pull/3614))
+
 ### Removals
 
 - Removed the PDF support for documentation, replaced it with the HTML output

charts/dex.yaml

@@ -14,6 +14,19 @@ tolerations:
 
 replicaCount: '__var__(dex.spec.deployment.replicas)'
 
+strategy:
+  rollingUpdate:
+    maxUnavailable: 1
+  type: RollingUpdate
+
+
+# NOTE: We use `__var_quoted__` as `affinity` is rendered as YAML
+# so result will be quoted
+affinity: '__var_quoted__(salt.metalk8s_service_configuration.get_pod_affinity(
+  dex.spec.deployment.affinity,
+  {"app.kubernetes.io/instance": "dex", "app.kubernetes.io/name": "dex"},
+  "metalk8s-auth"))'
+
 # grpc support
 grpc:
   enabled: false

charts/dex/templates/deployment.yaml

@@ -11,6 +11,10 @@ spec:
   selector:
     matchLabels:
       {{- include "dex.selectorLabels" . | nindent 6 }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   template:
     metadata:
       annotations:

charts/render.py

@@ -158,6 +158,13 @@ def replace_magic_strings(rendered_yaml):
         rendered_yaml,
     )
 
+    # Handle __var_quoted__
+    result = re.sub(
+        r"'__var_quoted__\((?P<varname>[^']*)\)'",
+        r"{% endraw -%}{{ \g<varname> }}{%- raw %}",
+        result,
+    )
+
     # Handle __var_tojson__
     result = re.sub(
         r"__var_tojson__\((?P<varname>[\w\-_]+(?:\.[\w\-_()|]+)*)\)",

docs/installation/bootstrap.rst

@@ -72,10 +72,11 @@ Configuration
             <feature_gate_name>: True
         coreDNS:
           replicas: 2
-          podAntiAffinity:
-            hard: []
-            soft:
-              - topologyKey: kubernetes.io/hostname
+          affinity:
+            podAntiAffinity:
+              hard: []
+              soft:
+                - topologyKey: kubernetes.io/hostname
 
 The ``networks`` field specifies a range of IP addresses written in CIDR
 notation for it's various subfields.
@@ -184,9 +185,10 @@ defaults kubernetes configuration.
 
       kubernetes:
         coreDNS:
-          podAntiAffinity:
-            hard:
-              - topologyKey: kubernetes.io/hostname
+          affinity:
+            podAntiAffinity:
+              hard:
+                - topologyKey: kubernetes.io/hostname
 
 .. _Feature Gates: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
 

eve/main.yml

@@ -346,7 +346,7 @@ models:
       env: &_env_bootstrap_config_ssh
         DEBUG: "%(prop:metalk8s_debug:-false)s"
         ARCHIVE: metalk8s.iso
-        COREDNS_ANTI_AFFINITY: "{}"
+        COREDNS_AFFINITY: "{}"
       command: |
         ssh -F ssh_config bootstrap "
         sudo bash << EOF
@@ -370,7 +370,7 @@ models:
         debug: ${DEBUG}
         kubernetes:
           coreDNS:
-            podAntiAffinity: ${COREDNS_ANTI_AFFINITY}
+            affinity: ${COREDNS_AFFINITY}
         END
         EOF"
       haltOnFailure: true
@@ -2359,8 +2359,8 @@ stages:
           env:
             <<: *_env_bootstrap_config_ssh
             ARCHIVE: /archives/metalk8s.iso
-            COREDNS_ANTI_AFFINITY: >-
-              {"hard": [{"topologyKey": "kubernetes.io/hostname"}]}
+            COREDNS_AFFINITY: >-
+              {"podAntiAffinity": {"hard": [{"topologyKey": "kubernetes.io/hostname"}]}}
       - ShellCommand:
           <<: *copy_iso_bootstrap_ssh
           env:
@@ -2491,7 +2491,12 @@ stages:
       - ShellCommand: *check_ssh_config_bootstrap
       - ShellCommand: *copy_bastion_priv_key_to_bootstrap
       - ShellCommand: *set_bootstrap_minion_id_ssh
-      - ShellCommand: *bootstrap_config_ssh
+      - ShellCommand:
+          <<: *bootstrap_config_ssh
+          env:
+            <<: *_env_bootstrap_config_ssh
+            COREDNS_AFFINITY: >-
+              {"podAntiAffinity": {"hard": [{"topologyKey": "kubernetes.io/hostname"}]}}
       - ShellCommand: *copy_iso_bootstrap_ssh
       - ShellCommand: *create_mountpoint_ssh
       - ShellCommand: *mount_iso_ssh

salt/_modules/metalk8s_service_configuration.py

@@ -95,3 +95,66 @@ def get_service_conf(
     )
 
     return merged_config
+
+
+def get_pod_affinity(affinities, label_selector, namespaces):
+    """Convert human readable affinity to real Kubernetes affinity
+
+    NOTE: This function only handle podAntiAffinity for the moment
+
+    Input look like
+    ```yaml
+    podAntiAffinity:
+        hard:
+            - topologyKey: kubernetes.io/hostname
+        soft:
+            - topologyKey: my.topology.key/important
+              weight: 100
+            - topologyKey: my.topology.key/a.bit.less.important
+    ```
+
+    Args:
+        value (dict): Simple dict affinity
+        label_selector (dict): Dict for label selector to match Pods for this affinity
+        namespaces (list): List of namespaces where those Pods sits
+
+    Returns:
+        A dict that can be used as `affinity` in a Kubernetes object
+    """
+    if not isinstance(namespaces, list):
+        namespaces = [namespaces]
+
+    result = {}
+    res_pod_anti_affinity = {}
+
+    pod_anti_affinity = (affinities or {}).get("podAntiAffinity") or {}
+
+    for soft_affinity in pod_anti_affinity.get("soft") or []:
+        res_pod_anti_affinity.setdefault(
+            "preferredDuringSchedulingIgnoredDuringExecution", []
+        ).append(
+            {
+                "weight": soft_affinity.get("weight", 1),
+                "podAffinityTerm": {
+                    "labelSelector": {"matchLabels": label_selector},
+                    "namespaces": namespaces,
+                    "topologyKey": soft_affinity["topologyKey"],
+                },
+            }
+        )
+
+    for hard_affinity in pod_anti_affinity.get("hard") or []:
+        res_pod_anti_affinity.setdefault(
+            "requiredDuringSchedulingIgnoredDuringExecution", []
+        ).append(
+            {
+                "labelSelector": {"matchLabels": label_selector},
+                "namespaces": namespaces,
+                "topologyKey": hard_affinity["topologyKey"],
+            }
+        )
+
+    if res_pod_anti_affinity:
+        result["podAntiAffinity"] = res_pod_anti_affinity
+
+    return result

salt/metalk8s/addons/dex/config/dex.yaml.j2

@@ -20,6 +20,14 @@ spec:
   # Deployment configuration
   deployment:
     replicas: 2
+    affinity:
+      podAntiAffinity:
+        soft:
+          - topologyKey: kubernetes.io/hostname
+        #   - topologyKey: my.second.important/topologyKey
+        #     weight: 42
+        # hard:
+        #   - topologyKey: kubernetes.io/hostname
 
   # Dex server configuration
   config:

salt/metalk8s/addons/dex/deployed/chart.sls

@@ -124,6 +124,10 @@ spec:
     matchLabels:
       app.kubernetes.io/instance: dex
       app.kubernetes.io/name: dex
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
   template:
     metadata:
       annotations:
@@ -133,6 +137,9 @@ spec:
         app.kubernetes.io/instance: dex
         app.kubernetes.io/name: dex
     spec:
+      affinity: {% endraw -%}{{ salt.metalk8s_service_configuration.get_pod_affinity(
+        dex.spec.deployment.affinity, {"app.kubernetes.io/instance": "dex", "app.kubernetes.io/name":
+        "dex"}, "metalk8s-auth") }}{%- raw %}
       containers:
       - args:
         - dex

salt/metalk8s/kubernetes/coredns/deployed.sls

@@ -8,41 +8,17 @@
 {%- set replicas = pillar_coredns.get("replicas") or 2 %}
 {%- set label_selector = {"k8s-app": "kube-dns"} %}
 
-{%- set pillar_affinities = pillar_coredns.get("podAntiAffinity", {}) %}
+{%- set pillar_affinities = pillar_coredns.get("affinity", {}) %}
 {#- NOTE: The default podAntiAffinity is a soft anti-affinity on hostname #}
-{%- set soft_affinities = pillar_affinities.get("soft") or [{"topologyKey": "kubernetes.io/hostname"}] %}
-{%- set hard_affinities = pillar_affinities.get("hard") or [] %}
-
-{%- set affinity = {
-  "podAntiAffinity": {}
-} %}
-
-{%- for soft_affinity in soft_affinities %}
-  {%- do affinity["podAntiAffinity"].setdefault(
-    "preferredDuringSchedulingIgnoredDuringExecution", []
-  ).append({
-    "weight": soft_affinity.get("weight", 1),
-    "podAffinityTerm": {
-      "labelSelector": {
-        "matchLabels": label_selector
-      },
-      "namespaces": ["kube-system"],
-      "topologyKey": soft_affinity["topologyKey"]
-    }
-  }) %}
-{%- endfor %}
-{%- for hard_affinity in hard_affinities %}
-  {%- do affinity["podAntiAffinity"].setdefault(
-    "requiredDuringSchedulingIgnoredDuringExecution", []
-  ).append({
-    "labelSelector": {
-      "matchLabels": label_selector
-    },
-    "namespaces": ["kube-system"],
-    "topologyKey": hard_affinity["topologyKey"]
-  }) %}
-{%- endfor %}
+{%- do pillar_affinities.setdefault("podAntiAffinity", {}).setdefault(
+    "soft", [{"topologyKey": "kubernetes.io/hostname"}]
+) %}
 
+{%- set affinity = salt.metalk8s_service_configuration.get_pod_affinity(
+    pillar_affinities,
+    label_selector,
+    "kube-system"
+) %}
 
 Create coredns ConfigMap:
   metalk8s_kubernetes.object_present:

salt/tests/unit/formulas/config.yaml

@@ -189,38 +189,42 @@ metalk8s:
             pillar_overrides:
               kubernetes:
                 coreDNS:
-                  podAntiAffinity:
-                    soft:
-                      - topologyKey: kubernetes.io/hostname
-                      - topologyKey: kubernetes.io/zone
-                        weight: 10
+                  affinity:
+                    podAntiAffinity:
+                      soft:
+                        - topologyKey: kubernetes.io/hostname
+                        - topologyKey: kubernetes.io/zone
+                          weight: 10
           "Hard anti-affinity on hostname":
             pillar_overrides:
               kubernetes:
                 coreDNS:
-                  podAntiAffinity:
-                    hard:
-                      - topologyKey: kubernetes.io/hostname
+                  affinity:
+                    podAntiAffinity:
+                      hard:
+                        - topologyKey: kubernetes.io/hostname
           "Multiple hard anti-affinity":
             pillar_overrides:
               kubernetes:
                 coreDNS:
-                  podAntiAffinity:
-                    hard:
-                      - topologyKey: kubernetes.io/hostname
-                      - topologyKey: kubernetes.io/zone
+                  affinity:
+                    podAntiAffinity:
+                      hard:
+                        - topologyKey: kubernetes.io/hostname
+                        - topologyKey: kubernetes.io/zone
           "Multiple hard and soft anti-affinity":
             pillar_overrides:
               kubernetes:
                 coreDNS:
-                  podAntiAffinity:
-                    soft:
-                      - topologyKey: kubernetes.io/hostname
-                      - topologyKey: kubernetes.io/zone
-                        weight: 10
-                    hard:
-                      - topologyKey: kubernetes.io/hostname
-                      - topologyKey: kubernetes.io/zone
+                  affinity:
+                    podAntiAffinity:
+                      soft:
+                        - topologyKey: kubernetes.io/hostname
+                        - topologyKey: kubernetes.io/zone
+                          weight: 10
+                      hard:
+                        - topologyKey: kubernetes.io/hostname
+                        - topologyKey: kubernetes.io/zone
 
       files:
         coredns-deployment.yaml.j2:

salt/tests/unit/formulas/fixtures/salt.py

@@ -9,6 +9,8 @@
 from typing import Any, Callable, Dict, List, Optional, Type
 from unittest.mock import MagicMock
 
+from _modules import metalk8s_service_configuration  # type: ignore
+
 import jinja2
 import pytest
 import salt.utils.data  # type: ignore
@@ -448,6 +450,10 @@ def metalk8s_network_get_ip_from_cidrs(
 register_basic("metalk8s_service_configuration.get_service_conf")(
     lambda _namespace, _name, defaults: defaults
 )
+# NOTE: This is a static function that does not rely on any salt stuff
+register_basic("metalk8s_service_configuration.get_pod_affinity")(
+    metalk8s_service_configuration.get_pod_affinity
+)
 
 # Used in metalk8s.salt.master.installed to mount Solution ISOs in the salt-master Pod.
 register_basic("metalk8s_solutions.list_available")(MagicMock(return_value={}))