Parametrize gateway topology deployment

Add "gateways: 2" parameter to ".shipyard*" file in order to
parametrize the amount of nodes that should be labeld as
gateway nodes.
In case of the label missing, both on the cluster or global level, the
GW node label will be applied automatically on single node.

Example of configuration:
---
submariner: true
nodes: control-plane worker
clusters:
  cluster1:
  cluster2:
    nodes: control-plane worker worker
    gateways: 2

In the above example, the first cluster will get 1 gw node applied and
second cluster will have 2 gw nodes applied.

Signed-off-by: Maxim Babushkin <[email protected]>

.github/workflows/verify.yml

@@ -0,0 +1,20 @@
+---
+name: Verify
+
+on:
+  pull_request:
+
+permissions: {}
+
+env:
+  DEBUG_PRINT: true
+jobs:
+  verify:
+    name: Verify gateway nodes
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
+
+      - name: Deploy and validate gateways custom topology
+        run: make script-test SCRIPT_TEST_ARGS="test/scripts/validation/test.sh"

scripts/shared/cloud-prepare.sh

@@ -28,15 +28,19 @@ function check_gateway_exists() {
 }
 
 function prepare_kind() {
-    read -r -a nodes <<< "${cluster_nodes[$cluster]}"
-    local node=${cluster}-${nodes[-1]}
-    kubectl label node "$node" "$GATEWAY_LABEL" --overwrite
-
-    if [[ "$AIR_GAPPED" = true ]]; then
-        local pub_ip
-        pub_ip=$(kubectl get nodes "$node" -o jsonpath="{.status.addresses[0].address}")
-        kubectl annotate node "$node" gateway.submariner.io/public-ip=ipv4:"$pub_ip"
-    fi
+    local gw_count="${cluster_gateways[$cluster]:-1}"
+
+    readarray -t nodes < <(kubectl get nodes --no-headers -o custom-columns=NAME:".metadata.name" | tac)
+
+    for node in "${nodes[@]:0:$gw_count}"; do
+        kubectl label node "$node" "$GATEWAY_LABEL" --overwrite
+
+        if [[ "$AIR_GAPPED" = true ]]; then
+            local pub_ip
+            pub_ip=$(kubectl get nodes "$node" -o jsonpath="{.status.addresses[0].address}")
+            kubectl annotate node "$node" gateway.submariner.io/public-ip=ipv4:"$pub_ip"
+        fi
+    done
 }
 
 function prepare_ocp() {

scripts/shared/lib/utils

@@ -218,7 +218,7 @@ function load_settings() {
     local DEBUG_PRINT=false
     expect_env SETTINGS "Deployment settings file"
     declare -ga clusters
-    declare -gA cluster_cni cluster_nodes cluster_subm
+    declare -gA cluster_cni cluster_nodes cluster_subm cluster_gateways
 
     local cluster_count
     cluster_count=$(_yq ".cluster-count")
@@ -239,6 +239,7 @@ function load_settings() {
         cluster_nodes["${cluster}"]=$(_yq ".clusters.${cluster}.nodes // .nodes")
         cluster_subm["${cluster}"]=$(_yq ".clusters.${cluster}.submariner")
         cluster_subm["${cluster}"]=$(_yq ".submariner")
+        cluster_gateways["${cluster}"]=$(_yq ".clusters.${cluster}.gateways // .gateways")
     done
 
     cat << EOM
@@ -248,6 +249,7 @@ Cluster settings::
   cni - $(typeset -p cluster_cni | cut -f 2- -d=)
   nodes per cluster - $(typeset -p cluster_nodes | cut -f 2- -d=)
   install submariner - $(typeset -p cluster_subm | cut -f 2- -d=)
+  label cluster gateways - $(typeset -p cluster_gateways | cut -f 2- -d=)
 EOM
 }
 

test/scripts/validation/scenario_config1

@@ -0,0 +1,6 @@
+---
+submariner: true
+nodes: control-plane worker
+clusters:
+  cluster1:
+    nodes: control-plane worker worker

test/scripts/validation/scenario_config2

@@ -0,0 +1,7 @@
+---
+submariner: true
+nodes: control-plane worker
+clusters:
+  cluster1:
+    nodes: control-plane worker worker
+    gateways: 2

test/scripts/validation/test.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -e
+
+source ${SCRIPTS_DIR}/lib/debug_functions
+source ${SCRIPTS_DIR}/lib/utils
+# source ${SCRIPTS_DIR}/cloud-prepare.sh
+
+function verify_gw_topology() {
+    local gateways_count
+    gateways="${cluster_gateways[$cluster]:-1}"
+
+    echo "Verify gateway nodes"
+    gateways_count=$(kubectl get nodes -l=submariner.io/gateway=true \
+        --no-headers=true -o custom-columns=NAME:.metadata.name | wc -l)
+
+    if [[ "$gateways" -ne "$gateways_count" ]]; then
+        echo "Expect $gateways gateways nodes but detected $gateways_count"
+        return 1
+    else
+        echo "Found expected number of gateways - $gateways"
+    fi
+}
+
+function remove_gw_labels() {
+    local gw_nodes
+
+    echo "Reset gateway nodes (unlabel)"
+    readarray -t gw_nodes < <(kubectl get nodes -l=submariner.io/gateway=true \
+        --no-headers=true -o custom-columns=NAME:.metadata.name)
+
+    for node in "${gw_nodes[@]}"; do
+        kubectl label --overwrite nodes "$node" submariner.io/gateway-
+    done
+}
+
+echo "Prepare cluster"
+make clusters SETTINGS=test/scripts/validation/scenario_config1
+
+for scenario in scenario_config1 scenario_config2 scenario_config3; do
+    export SETTINGS="test/scripts/validation/$scenario"
+    echo "Set gateway nodes according to $scenario scenario"
+    make cloud-prepare
+
+    declare_kubeconfig
+    load_settings
+    clusters=($(kind get clusters))
+    run_parallel "${clusters[*]}" verify_gw_topology
+    run_parallel "${clusters[*]}" remove_gw_labels
+done
+
+echo "Perform cleanup"
+make cleanup SETTINGS=test/scripts/validation/scenario_config1