salt,tests,docs: Add podAntiAffinity support for CoreDNS

Add ability for the user to change podAntiAffinity for CoreDNS
deployment and also have a default soft podAntiAffinity on hostname so
that if it's possible each CoreDNS replica will sit on a different node
by default.
Trigger a rollout restart of CoreDNS deployment after deploying a new
infra node in order to "apply" soft anti-affinity if possible

Fixes: #3574

CHANGELOG.md

@@ -6,6 +6,12 @@
   when observing the cluster state (used in the UI Dashboard page)
   (PR[#3540](https://github.com/scality/metalk8s/pull/3540))
 
+- [#3574](https://github.com/scality/metalk8s/issues/3574) - Allow to manage
+  soft and hard `podAntiAffinity` for `CoreDNS` from Bootstrap configuration
+  file, with a default soft anti-affinity on hostname, so that if it's
+  possible each `CoreDNS` pods will sit on different infra node
+  (PR[#3579](https://github.com/scality/metalk8s/pull/3579))
+
 ### Removals
 
 - Removed the PDF support for documentation, replaced it with the HTML output

docs/installation/bootstrap.rst

@@ -70,6 +70,11 @@ Configuration
         apiServer:
           featureGates:
             <feature_gate_name>: True
+        coreDNS:
+          podAntiAffinity:
+            hard: []
+            soft:
+              - topologyKey: kubernetes.io/hostname
 
 The ``networks`` field specifies a range of IP addresses written in CIDR
 notation for it's various subfields.
@@ -159,10 +164,27 @@ the bootstrap script is executed, those ISOs are automatically mounted and the
 system is configured to re-mount them automatically after a reboot.
 
 The ``kubernetes`` field can be omitted if you do not have any specific
-Kubernetes `Feature Gates`_ to enable or disable.
-If you need to enable or disable specific features for ``kube-apiserver``
-configure the corresponding entries in the
-``kubernetes.apiServer.featureGates`` mapping.
+Kubernetes `Feature Gates`_ to enable or disable and if you are ok with
+defaults kubernetes configuration.
+
+  If you need to enable or disable specific features for ``kube-apiserver``
+  configure the corresponding entries in the
+  ``kubernetes.apiServer.featureGates`` mapping.
+
+  If you want to override the default ``coreDNS`` podAntiAffinity, by default
+  MetalK8s use soft podAntiAffinity on hostname so that if it's possible
+  ``coreDNS`` pods will be spread on different infra nodes.
+  If you have more infra node than ``coreDNS`` replicas (default is 2), you
+  should set hard podAntiAffinity on hostname so that you are sure that
+  ``coreDNS`` pods sit on different node, to do so:
+
+    .. code-block:: yaml
+
+      kubernetes:
+        coreDNS:
+          podAntiAffinity:
+            hard:
+              - topologyKey: kubernetes.io/hostname
 
 .. _Feature Gates: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
 

eve/main.yml

@@ -346,6 +346,7 @@ models:
       env: &_env_bootstrap_config_ssh
         DEBUG: "%(prop:metalk8s_debug:-false)s"
         ARCHIVE: metalk8s.iso
+        COREDNS_ANTI_AFFINITY: "{}"
       command: |
         ssh -F ssh_config bootstrap "
         sudo bash << EOF
@@ -367,6 +368,9 @@ models:
         archives:
           - \"\$(readlink -f \"${ARCHIVE}\")\"
         debug: ${DEBUG}
+        kubernetes:
+          coreDNS:
+            podAntiAffinity: ${COREDNS_ANTI_AFFINITY}
         END
         EOF"
       haltOnFailure: true
@@ -2355,6 +2359,8 @@ stages:
           env:
             <<: *_env_bootstrap_config_ssh
             ARCHIVE: /archives/metalk8s.iso
+            COREDNS_ANTI_AFFINITY: >-
+              {"hard": [{"topologyKey": "kubernetes.io/hostname"}]}
       - ShellCommand:
           <<: *copy_iso_bootstrap_ssh
           env:

salt/metalk8s/kubernetes/coredns/deployed.sls

@@ -3,6 +3,44 @@
 
 {%- set cluster_dns_ip = salt.metalk8s_network.get_cluster_dns_ip() %}
 
+{%- set label_selector = {"k8s-app": "kube-dns"} %}
+
+{%- set pillar_affinities = pillar.kubernetes.get("coreDNS", {}).get("podAntiAffinity", {}) %}
+{#- NOTE: The default podAntiAffinity is a soft anti-affinity on hostname #}
+{%- set soft_affinities = pillar_affinities.get("soft") or [{"topologyKey": "kubernetes.io/hostname"}] %}
+{%- set hard_affinities = pillar_affinities.get("hard") or [] %}
+
+{%- set affinity = {
+  "podAntiAffinity": {}
+} %}
+
+{%- for soft_affinity in soft_affinities %}
+  {%- do affinity["podAntiAffinity"].setdefault(
+    "preferredDuringSchedulingIgnoredDuringExecution", []
+  ).append({
+    "weight": soft_affinity.get("weight", 1),
+    "podAffinityTerm": {
+      "labelSelector": {
+        "matchLabels": label_selector
+      },
+      "namespaces": ["kube-system"],
+      "topologyKey": soft_affinity["topologyKey"]
+    }
+  }) %}
+{%- endfor %}
+{%- for hard_affinity in hard_affinities %}
+  {%- do affinity["podAntiAffinity"].setdefault(
+    "requiredDuringSchedulingIgnoredDuringExecution", []
+  ).append({
+    "labelSelector": {
+      "matchLabels": label_selector
+    },
+    "namespaces": ["kube-system"],
+    "topologyKey": hard_affinity["topologyKey"]
+  }) %}
+{%- endfor %}
+
+
 Create coredns ConfigMap:
   metalk8s_kubernetes.object_present:
     - manifest:
@@ -41,6 +79,10 @@ Create coredns deployment:
   metalk8s_kubernetes.object_present:
     - name: salt://{{ slspath }}/files/coredns-deployment.yaml.j2
     - template: jinja
+    - defaults:
+        label_selector: {{ label_selector | tojson }}
+        affinity: {{ affinity | tojson }}
+
     - require:
       - metalk8s_kubernetes: Create coredns ConfigMap
 

salt/metalk8s/kubernetes/coredns/files/coredns-deployment.yaml.j2

@@ -20,12 +20,14 @@ spec:
       maxUnavailable: 1
   selector:
     matchLabels:
-      k8s-app: kube-dns
+      {{ label_selector | yaml(False) | indent(8) }}
   template:
     metadata:
       labels:
-        k8s-app: kube-dns
+        {{ label_selector | yaml(False) | indent(8) }}
     spec:
+      affinity:
+        {{ affinity | yaml(False) | indent(8) }}
       priorityClassName: system-cluster-critical
       tolerations:
         - key: "CriticalAddonsOnly"

salt/metalk8s/orchestrate/deploy_node.sls

@@ -307,3 +307,18 @@ Kill kube-controller-manager on all master nodes:
         pattern: kube-controller-manager
     - require:
       - salt: Run the highstate
+
+{%- if 'infra' in roles and 'infra' not in skip_roles %}
+
+# Trigger a restart of CoreDNS pods so that "soft anti-affinity" can be applied
+Restart CoreDNS pods:
+  module.run:
+    - metalk8s_kubernetes.rollout_restart:
+      - name: coredns
+      - namespace: kube-system
+      - kind: Deployment
+      - apiVersion: apps/v1
+    - require:
+      - metalk8s_cordon: Uncordon the node
+
+{%- endif %}

salt/tests/unit/formulas/config.yaml

@@ -181,6 +181,66 @@ metalk8s:
                 config_digest: abcdefgh12345
                 metalk8s_version: "2.7.1"
 
+    coredns:
+      deployed.sls:
+        _cases:
+          "Simple hostname soft anti-affinity (default)": {}
+          "Multiple soft anti-affinity":
+            pillar_overrides:
+              kubernetes:
+                coreDNS:
+                  podAntiAffinity:
+                    soft:
+                      - topologyKey: kubernetes.io/hostname
+                      - topologyKey: kubernetes.io/zone
+                        weight: 10
+          "Hard anti-affinity on hostname":
+            pillar_overrides:
+              kubernetes:
+                coreDNS:
+                  podAntiAffinity:
+                    hard:
+                      - topologyKey: kubernetes.io/hostname
+          "Multiple hard anti-affinity":
+            pillar_overrides:
+              kubernetes:
+                coreDNS:
+                  podAntiAffinity:
+                    hard:
+                      - topologyKey: kubernetes.io/hostname
+                      - topologyKey: kubernetes.io/zone
+          "Multiple hard and soft anti-affinity":
+            pillar_overrides:
+              kubernetes:
+                coreDNS:
+                  podAntiAffinity:
+                    soft:
+                      - topologyKey: kubernetes.io/hostname
+                      - topologyKey: kubernetes.io/zone
+                        weight: 10
+                    hard:
+                      - topologyKey: kubernetes.io/hostname
+                      - topologyKey: kubernetes.io/zone
+
+      files:
+        coredns-deployment.yaml.j2:
+          _cases:
+            "From metalk8s.kubernetes.coredns.deployed":
+              extra_context:
+                label_selector:
+                  k8s-app: kube-dns
+                affinity:
+                  podAntiAffinity:
+                    preferredDuringSchedulingIgnoredDuringExecution:
+                      - weight: 1
+                        podAffinityTerm:
+                          labelSelector:
+                            matchLabels:
+                              k8s-app: kube-dns
+                          namespaces:
+                            - kube-system
+                          topologyKey: kubernetes.io/hostname
+
     etcd:
       files:
         manifest.yaml.j2:

tests/post/features/dns_resolution.feature

@@ -3,3 +3,9 @@ Feature: CoreDNS resolution
     Scenario: check DNS
         Given pods with label 'k8s-app=kube-dns' are 'Ready'
         Then the hostname 'kubernetes.default' should be resolved
+
+    Scenario: DNS pods spreading
+        Given the Kubernetes API is available
+        And we are on a multi node cluster
+        Then pods with label 'k8s-app=kube-dns' are 'Ready'
+        And each pods with label 'k8s-app=kube-dns' are on a different node

tests/post/steps/conftest.py

@@ -135,4 +135,18 @@ def then_check_pod_status(request, host, k8s_client, label, expected_status):
     _check_pods_status(k8s_client, expected_status, ssh_config, label=label)
 
 
+@then(parsers.parse("each pods with label '{selector}' are on a different node"))
+def then_check_pod_different_node(ssh_config, host, k8s_client, selector):
+    pods = kube_utils.get_pods(k8s_client, ssh_config, selector)
+    assert pods
+
+    nodes = set()
+
+    for pod in pods:
+        assert (
+            pod.spec.nodeName not in nodes
+        ), f"Node '{pod.spec.nodeName}' has several Pod with label '{selector}'"
+        nodes.add(pod.spec.nodeName)
+
+
 # }}}

tests/post/steps/test_dns.py

@@ -53,6 +53,11 @@ def test_dns(host):
     pass
 
 
+@scenario("../features/dns_resolution.feature", "DNS pods spreading")
+def test_dns_spread(host):
+    pass
+
+
 @then(parsers.parse("the hostname '{hostname}' should be resolved"))
 def resolve_hostname(utils_pod, host, hostname):
     with host.sudo():