charts,salt: Allow to set Control Plane Ingress IP to an external IP

CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 ## Release 2.11.7 (in development)
+### Enhancements
+
+- Allow to set Control Plane Ingress IP to an external IP (like
+  a load balancer IP)
+  (PR[#3752](https://github.com/scality/metalk8s/pull/3752))
 
 ## Release 2.11.6
 ### Enhancements

charts/ingress-nginx-control-plane-daemonset.yaml

@@ -38,8 +38,7 @@ controller:
   service:
     type: ClusterIP
 
-    externalIPs:
-    - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw -%}'
+    externalIPs: '__var_tojson__(salt.metalk8s_network.get_control_plane_ingress_external_ips())'
 
     enableHttp: false
 

charts/render.py

@@ -168,7 +168,7 @@ def replace_magic_strings(rendered_yaml):
     # Handle __var_tojson__
     result = re.sub(
         r"__var_tojson__\((?P<varname>[\w\-_]+(?:\.[\w\-_()|]+)*)\)",
-        r"  {% endraw -%}{{ \g<varname> | tojson }}{%- raw %}",
+        r"{% endraw -%}{{ \g<varname> | tojson }}{%- raw %}",
         result,
     )
 

salt/_modules/metalk8s_network.py

@@ -257,3 +257,25 @@ def get_control_plane_ingress_endpoint():
     return "https://{}:8443".format(
         __salt__["metalk8s_network.get_control_plane_ingress_ip"]()
     )
+
+
+def get_control_plane_ingress_external_ips():
+    """Get all Control Plane Ingress external IPs
+
+    NOTE: Only the first one will be used as redirect IP for oidc
+    """
+    master_nodes = __salt__["metalk8s.minions_by_role"]("master")
+
+    # This function only run on master
+    mine_ret = __salt__["saltutil.runner"](
+        "mine.get", tgt=",".join(master_nodes), tgt_type="list", fun="control_plane_ip"
+    )
+
+    if not isinstance(mine_ret, dict):
+        raise CommandExecutionError(
+            f"Unable to get master Control Plane IPs: {mine_ret}"
+        )
+
+    return [__salt__["metalk8s_network.get_control_plane_ingress_ip"]()] + sorted(
+        list(mine_ret.values())
+    )

salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-daemonset.sls

@@ -280,9 +280,7 @@ metadata:
   name: ingress-nginx-control-plane-controller
   namespace: metalk8s-ingress
 spec:
-  externalIPs:
-  - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw
-    -%}'
+  externalIPs: {% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_external_ips() | tojson }}{%- raw %}
   ipFamilies:
   - IPv4
   ipFamilyPolicy: SingleStack

salt/metalk8s/orchestrate/deploy_node.sls

@@ -326,3 +326,18 @@ Restart CoreDNS pods:
       - metalk8s_cordon: Uncordon the node
 
 {%- endif %}
+
+{%- if 'master' in roles and 'master' not in skip_roles %}
+
+# Trigger a reconfiguration of the Control Plane Ingress Controller as we
+# want to expose the Controller on every master node Control Plane IP
+Reconfigure Control Plane Ingress:
+  salt.runner:
+    - name: state.orchestrate
+    - mods:
+      - metalk8s.addons.nginx-ingress-control-plane.deployed
+    - saltenv: {{ saltenv }}
+    - require:
+      - metalk8s_cordon: Uncordon the node
+
+{%- endif %}

salt/tests/unit/formulas/fixtures/salt.py

@@ -424,6 +424,16 @@ def metalk8s_grafana_load_dashboard(source: str, **_kwargs: Any) -> Any:
 register_basic("metalk8s_network.get_control_plane_ingress_endpoint")(
     MagicMock(return_value="https://192.168.1.240:8443")
 )
+register_basic("metalk8s_network.get_control_plane_ingress_external_ips")(
+    MagicMock(
+        return_value=[
+            "192.168.1.240",
+            "192.168.1.100",
+            "192.168.1.101",
+            "192.168.1.102",
+        ]
+    )
+)
 
 
 @register_basic("metalk8s_network.get_ip_from_cidrs")

salt/tests/unit/modules/files/test_metalk8s_network.yaml

@@ -135,3 +135,87 @@ get_control_plane_ingress_endpoint:
   - cp_ingress_ip_ret: "Error because of banana"
     raises: true
     result: "Error because of banana"
+
+get_control_plane_ingress_external_ips:
+  # 1. Nominal single node (using bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.1
+    mine_ret:
+      bootstrap: 1.1.1.1
+    result:
+      - 1.1.1.1
+      - 1.1.1.1
+
+  # 2. Nominal single node (using non-bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.4
+    mine_ret:
+      bootstrap: 1.1.1.1
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+
+  # 3. Nominal multi node (using bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.1
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.1
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 4. Nominal multi node (using non-node IP)
+  - cp_ingress_ip_ret: 1.1.1.4
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 5. Nominal multi node (using non-bootstrap node IP)
+  - cp_ingress_ip_ret: 1.1.1.2
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.2
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 6. Multi node, one master node not yet in mine
+  - cp_ingress_ip_ret: 1.1.1.4
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+      - 1.1.1.3
+
+  # 7. Error unable to get from mine
+  - mine_ret: "ErROr"
+    raises: true
+    result: "Unable to get master Control Plane IPs: ErROr"

salt/tests/unit/modules/test_metalk8s_network.py

@@ -363,3 +363,39 @@ def test_get_control_plane_ingress_endpoint(
                 self.assertEqual(
                     metalk8s_network.get_control_plane_ingress_endpoint(), result
                 )
+
+    @utils.parameterized_from_cases(
+        YAML_TESTS_CASES["get_control_plane_ingress_external_ips"]
+    )
+    def test_get_control_plane_ingress_external_ips(
+        self,
+        result,
+        raises=False,
+        cp_ingress_ip_ret=None,
+        master_nodes_ret=None,
+        mine_ret=None,
+    ):
+        """
+        Tests the return of `get_control_plane_ingress_external_ips` function
+        """
+        salt_dict = {
+            "metalk8s_network.get_control_plane_ingress_ip": MagicMock(
+                return_value=cp_ingress_ip_ret
+            ),
+            "metalk8s.minions_by_role": MagicMock(
+                return_value=master_nodes_ret or ["bootstrap"]
+            ),
+            "saltutil.runner": MagicMock(return_value=mine_ret),
+        }
+
+        with patch.dict(metalk8s_network.__salt__, salt_dict):
+            if raises:
+                self.assertRaisesRegex(
+                    CommandExecutionError,
+                    result,
+                    metalk8s_network.get_control_plane_ingress_external_ips,
+                )
+            else:
+                self.assertEqual(
+                    metalk8s_network.get_control_plane_ingress_external_ips(), result
+                )