chart,salt,tests,docs: Add metallb deployment in MetalK8s deployment

As part of MetalK8s we will use metallb for control plane ingress
if this one is enabled in the Bootstrap Config

Chart get rendered with this command:
```
./charts/render.py metallb --namespace metalk8s-loadbalancing \
  charts/metallb.yaml charts/metallb/ \
  > salt/metalk8s/addons/metallb/deployed/chart.sls
```

NOTE: When we use metallb we do not need to use Nginx Ingress as a
DaemonSet, instead we use a Deployment

Nginx Ingress Control Plane deployment get rendered with this command:
```
./charts/render.py ingress-nginx-control-plane --namespace metalk8s-ingress \
  charts/ingress-nginx-control-plane-deployment.yaml charts/ingress-nginx/ \
  > salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-deployment.sls
```

Fixes: #2381

CHANGELOG.md

@@ -66,6 +66,11 @@
   nodes. This failover is not managed by MetalK8s.
   (PR[#3415](https://github.com/scality/metalk8s/pull/3415))
 
+- [#2381](https://github.com/scality/metalk8s/issues/2381)) - Allow
+  MetalK8s to manage Control Plane Ingress Virtual IP using MetalLB if
+  it possible in the user environment
+  (PR[#3418](https://github.com/scality/metalk8s/pull/3418))
+
 ### Breaking changes
 
 - [#2199](https://github.com/scality/metalk8s/issues/2199) - Prometheus label

buildchain/buildchain/constants.py

@@ -17,6 +17,7 @@
 CMD_WIDTH: int = 14
 
 # URLs of the main container repositories.
+BITNAMI_REPOSITORY: str = "docker.io/bitnami"
 CALICO_REPOSITORY: str = "docker.io/calico"
 COREDNS_REPOSITORY: str = "k8s.gcr.io/coredns"
 COREOS_REPOSITORY: str = "quay.io/coreos"

buildchain/buildchain/image.py

@@ -174,6 +174,7 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage:
 TO_PULL: List[targets.RemoteImage] = []
 
 IMGS_PER_REPOSITORY: Dict[str, List[str]] = {
+    constants.BITNAMI_REPOSITORY: ["metallb-controller", "metallb-speaker"],
     constants.CALICO_REPOSITORY: [
         "calico-node",
         "calico-kube-controllers",

buildchain/buildchain/salt_tree.py

@@ -429,7 +429,15 @@ def _get_parts(self) -> Iterator[str]:
         "salt/metalk8s/addons/nginx-ingress-control-plane/deployed/",
         "chart-daemonset.sls",
     ),
+    Path(
+        "salt/metalk8s/addons/nginx-ingress-control-plane/deployed/",
+        "chart-deployment.sls",
+    ),
     Path("salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/chart.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/config.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/init.sls"),
+    Path("salt/metalk8s/addons/metallb/deployed/namespace.sls"),
     Path("salt/metalk8s/beacon/certificates.sls"),
     Path("salt/metalk8s/container-engine/containerd/configured.sls"),
     Path("salt/metalk8s/container-engine/containerd/files/50-metalk8s.conf.j2"),

buildchain/buildchain/versions.py

@@ -175,6 +175,16 @@ def _version_prefix(version: str, prefix: str = "v") -> str:
         version="v1.1.2",
         digest="sha256:22fbde17ab647ddf89841e5e464464eece111402b7d599882c2a3393bc0d2810",
     ),
+    Image(
+        name="metallb-controller",
+        version="0.9.6-debian-10-r52",
+        digest="sha256:a493e311beb663c7282fe6c3712899ab6fb7bd0ad4a38388ea1b97d7d735ff8a",
+    ),
+    Image(
+        name="metallb-speaker",
+        version="0.9.6-debian-10-r54",
+        digest="sha256:27f94679069f184d9bc67f6d5eccfc961be7588a73a066a8a4f87b6b3ef75614",
+    ),
     Image(
         name="pause",
         version="3.2",

charts/ingress-nginx-control-plane-deployment.yaml

@@ -0,0 +1,76 @@
+controller:
+  image:
+    digest: null
+    repository: '__image__(nginx-ingress-controller)'
+
+  defaultBackendService: 'metalk8s-ingress/nginx-ingress-default-backend'
+
+  electionID: ingress-control-plane-controller-leader
+
+  ingressClass: nginx-control-plane
+
+  admissionWebhooks:
+    enabled: false
+
+  kind: Deployment
+
+  replicaCount: 2
+
+  minAvailable: 0
+
+  updateStrategy:
+    type: RollingUpdate
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  service:
+    loadBalancerIP: '__var__(salt.metalk8s_network.get_control_plane_ingress_ip())'
+    externalTrafficPolicy: Local
+
+    enableHttp: false
+
+    ports:
+      https: 8443
+
+  extraArgs:
+    default-ssl-certificate: "metalk8s-ingress/ingress-control-plane-default-certificate"
+    metrics-per-host: false
+
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        metalk8s.scality.com/monitor: ''
+
+defaultBackend:
+  enabled: true
+
+  image:
+    repository: '__image__(nginx-ingress-defaultbackend-amd64)'
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''

charts/metallb.yaml

@@ -0,0 +1,42 @@
+existingConfigMap: metallb-config
+
+controller:
+  image:
+    registry: null
+    repository: '__image__(metallb-controller)'
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  podAnnotations:
+    checksum/config: '__slot__:salt:metalk8s_kubernetes.get_object_digest(kind="ConfigMap", apiVersion="v1", namespace="metalk8s-loadbalancing", name="metallb-config", path="data:config")'
+
+speaker:
+  image:
+    registry: null
+    repository: '__image__(metallb-speaker)'
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ''
+
+  tolerations:
+    - key: "node-role.kubernetes.io/bootstrap"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/infra"
+      operator: "Exists"
+      effect: "NoSchedule"

docs/installation/bootstrap.rst

@@ -49,6 +49,8 @@ Configuration
           cidr: <CIDR-notation>
           ingress:
             ip: <IP-for-ingress>
+          metalLB:
+            enabled: <boolean>
         workloadPlane:
           cidr: <CIDR-notation>
           mtu: <network-MTU>
@@ -92,6 +94,16 @@ notation for it's various subfields.
       that if you lose the Bootstrap node, you no longer have access to any
       control plane component).
 
+      This ``ip`` for ``ingress`` can be managed by MetalK8s directly if
+      it's possible in your environment, to do so we use
+      `MetalLB <https://metallb.universe.tf/>`_ that allow to manage this
+      Virtual IP directly on Layer2 using only
+      `ARP <https://en.wikipedia.org/wiki/Address_Resolution_Protocol>`_
+      requests, in order to be able to use MetalLB your network need to
+      properly broadcast ARP requests so that Control Plane node hosting
+      the Virtual IP can answer to this ARP request.
+      When MetalLB is enabled this ingress IP is mandatory.
+
       For ``workloadPlane`` entry an
       `MTU <https://en.wikipedia.org/wiki/Maximum_transmission_unit>`_ can
       also be provided, this MTU value should be the lowest MTU value accross

docs/operation/changing_control_plane_ingress_ip.rst

@@ -1,6 +1,15 @@
 Changing the Control Plane Ingress IP
 =====================================
 
+This procedure describes how to change the Control Plane Ingress IP, and
+to enable (or disable) MetalLB management of this IP.
+
+.. note::
+
+  Disabling MetalLB using this procedure does **not** remove MetalLB,
+  it simply disables its use for managing the ``LoadBalancer`` *Service*
+  for MetalK8s Control Plane Ingress.
+
 #. On the Bootstrap node, update the ``ip`` field from
    ``networks.controlPlane.ingress`` in the Bootstrap configuration file.
    (refer to :ref:`Bootstrap Configuration<Bootstrap Configuration>`)
@@ -18,6 +27,16 @@ Changing the Control Plane Ingress IP
       $ salt-call metalk8s_network.get_control_plane_ingress_ip
       local:
           <my-new-ip>
+      $ salt-call pillar.get networks:control_plane
+      local:
+        ----------
+        cidr:
+          - <control-plane-cidr>
+        ingress:
+          ip:
+            <my-new-ip>
+        metalLB:
+          enabled: <true | false>
 
 #. On the Bootstrap node, reconfigure apiServer:
 

eve/main.yml

@@ -356,6 +356,10 @@ models:
         networks:
           controlPlane:
             cidr: 192.168.1.0/24
+            metalLB:
+              enabled: true
+            ingress:
+              ip: 192.168.1.254
           workloadPlane:
             cidr: 192.168.2.0/24
         ca:
@@ -480,7 +484,7 @@ models:
       name: Run UI tests on Bastion
       env: &_env_bastion_ui_tests
         TEST_FILTER: "e2e"
-        TARGET_URL: "https://%(prop:bootstrap_control_plane_ip)s:8443"
+        TARGET_URL: "https://192.168.1.254:8443"
       command: >
         ssh -F ssh_config bastion --
         "cd metalk8s/ui &&

salt/_pillar/metalk8s.py

@@ -1,6 +1,7 @@
 import logging
 from collections import Mapping
 
+import salt.utils.dictupdate
 import salt.utils.files
 import salt.utils.yaml
 
@@ -90,6 +91,17 @@ def _load_networks(config_data):
             if not isinstance(networks_data[net]["cidr"], list):
                 networks_data[net]["cidr"] = [networks_data[net]["cidr"]]
 
+    # MetalLB disabled by default
+    networks_data["controlPlane"].setdefault("metalLB", {}).setdefault("enabled", False)
+
+    if networks_data["controlPlane"]["metalLB"]["enabled"] and not networks_data[
+        "controlPlane"
+    ].get("ingress", {}).get("ip"):
+        errors.append(
+            "'ip' for 'ingress' in 'controlPlane' network is mandatory when 'metalLB'"
+            "is enabled"
+        )
+
     if errors:
         return __utils__["pillar_utils.errors_to_dict"](errors)