Add multi-cluster kind setup for testing

scripts/.gitignore

@@ -0,0 +1 @@
+*.kubeconfig

scripts/cluster1.yml

@@ -0,0 +1,8 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  podSubnet: "10.10.0.0/16"
+  serviceSubnet: "10.11.0.0/16"
+nodes:
+  - role: control-plane
+    image: kindest/node:${KUBE_VERSION}

scripts/cluster2.yml

@@ -0,0 +1,8 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  podSubnet: "10.12.0.0/16"
+  serviceSubnet: "10.13.0.0/16"
+nodes:
+  - role: control-plane
+    image: kindest/node:${KUBE_VERSION}

scripts/cluster3.yml

@@ -0,0 +1,8 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  podSubnet: "10.14.0.0/16"
+  serviceSubnet: "10.15.0.0/16"
+nodes:
+  - role: control-plane
+    image: kindest/node:${KUBE_VERSION}

scripts/cluster4.yml

@@ -0,0 +1,8 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  podSubnet: "10.16.0.0/16"
+  serviceSubnet: "10.17.0.0/16"
+nodes:
+  - role: control-plane
+    image: kindest/node:${KUBE_VERSION}

scripts/setup_e2e.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+
+# This source file is part of the FoundationDB open source project
+#
+# Copyright 2022 Apple Inc. and the FoundationDB project authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo errexit
+
+# wait_for_node_setup is used to wait until the kubelet has started and the Pod CIDR is available.
+function wait_for_node_setup() {
+  for _ in {1..60}; do
+    if [ -n "$(kubectl --kubeconfig "${1}" get node  -o jsonpath='{range .items[*]}{.spec.podCIDR}{"\n"}')" ]; then
+      return
+    fi
+    sleep 1
+  done
+
+  exit 1
+}
+
+function preload_foundationdb_images() {
+  kind load docker-image "foundationdb/foundationdb:${2}" --name "${1}"
+  kind load docker-image "foundationdb/foundationdb-kubernetes-sidecar:${2}-1" --name "${1}"
+}
+
+# add_routes is used to allow Pods in the different clusters to communicate
+function add_routes() {
+  unset IFS
+  routes=$(kubectl --kubeconfig "${3}" get node "${2}" -o jsonpath='ip route add {.spec.podCIDR} via {.status.addresses[?(.type=="InternalIP")].address}')
+  echo "Connecting cluster ${1} to ${2}"
+
+  IFS=$'\n'
+  for n in $(kind get nodes --name "${1}"); do
+    for r in $routes; do
+      docker exec "${n}" bash -c "${r}"
+    done
+   done
+   unset IFS
+}
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+cd "${SCRIPT_DIR}"
+
+# Kubernetes version for the Kind clusters
+KUBE_VERSION=${KUBE_VERSION:-"v1.24.7"}
+FDB_VERSION=${FDB_VERSION:-"7.1.25"}
+
+# To test a multi-region FDB cluster setup we need to have 4 Kubernetes clusters
+cluster1=${CLUSTER1:-cluster1}
+cluster2=${CLUSTER2:-cluster2}
+cluster3=${CLUSTER3:-cluster3}
+cluster4=${CLUSTER4:-cluster4}
+
+# For all the clusters we have to create tha according kubeconfig
+kubeconfig1=${KUBECONFIG1:-"${cluster1}.kubeconfig"}
+kubeconfig2=${KUBECONFIG2:-"${cluster2}.kubeconfig"}
+kubeconfig3=${KUBECONFIG3:-"${cluster3}.kubeconfig"}
+kubeconfig4=${KUBECONFIG4:-"${cluster4}.kubeconfig"}
+
+# We have to export them for envsubst
+export KUBE_VERSION
+export reg_name
+export reg_port
+
+# Create a cluster with the local registry enabled in containerd
+envsubst < "${cluster1}.yml" | kind create cluster --name "${cluster1}" --config=-
+envsubst < "${cluster2}.yml" | kind create cluster --name "${cluster2}" --config=-
+envsubst < "${cluster3}.yml" | kind create cluster --name "${cluster3}" --config=-
+envsubst < "${cluster4}.yml" | kind create cluster --name "${cluster4}" --config=-
+
+unset KUBE_VERSION reg_name reg_port
+
+kind get kubeconfig --name "${cluster1}" > "${kubeconfig1}"
+kind get kubeconfig --name "${cluster2}" > "${kubeconfig2}"
+kind get kubeconfig --name "${cluster3}" > "${kubeconfig3}"
+kind get kubeconfig --name "${cluster4}" > "${kubeconfig4}"
+
+echo "Preload FoundationDB images"
+docker pull "foundationdb/foundationdb:${FDB_VERSION}"
+docker pull "foundationdb/foundationdb-kubernetes-sidecar:${FDB_VERSION}-1"
+
+preload_foundationdb_images "${cluster1}" "${FDB_VERSION}"
+preload_foundationdb_images "${cluster2}" "${FDB_VERSION}"
+preload_foundationdb_images "${cluster3}" "${FDB_VERSION}"
+preload_foundationdb_images "${cluster4}" "${FDB_VERSION}"
+
+echo "Waiting for Kind nodes to be initialized"
+wait_for_node_setup "${kubeconfig1}"
+wait_for_node_setup "${kubeconfig2}"
+wait_for_node_setup "${kubeconfig3}"
+wait_for_node_setup "${kubeconfig4}"
+
+echo "Connect Kind clusters"
+add_routes "${cluster1}" "${cluster2}-control-plane" "${kubeconfig2}"
+add_routes "${cluster1}" "${cluster3}-control-plane" "${kubeconfig3}"
+add_routes "${cluster1}" "${cluster4}-control-plane" "${kubeconfig4}"
+
+add_routes "${cluster2}" "${cluster1}-control-plane" "${kubeconfig1}"
+add_routes "${cluster2}" "${cluster3}-control-plane" "${kubeconfig3}"
+add_routes "${cluster2}" "${cluster4}-control-plane" "${kubeconfig4}"
+
+add_routes "${cluster3}" "${cluster1}-control-plane" "${kubeconfig1}"
+add_routes "${cluster3}" "${cluster2}-control-plane" "${kubeconfig2}"
+add_routes "${cluster3}" "${cluster4}-control-plane" "${kubeconfig4}"
+
+add_routes "${cluster4}" "${cluster1}-control-plane" "${kubeconfig1}"
+add_routes "${cluster4}" "${cluster2}-control-plane" "${kubeconfig2}"
+add_routes "${cluster4}" "${cluster3}-control-plane" "${kubeconfig3}"
+
+echo "Install the CRDs in every Kind cluster"
+kubectl --kubeconfig "${kubeconfig1}" apply -f ../config/crd/bases
+kubectl --kubeconfig "${kubeconfig2}" apply -f ../config/crd/bases
+kubectl --kubeconfig "${kubeconfig3}" apply -f ../config/crd/bases
+kubectl --kubeconfig "${kubeconfig4}" apply -f ../config/crd/bases
+
+# TODO(johscheuer): Make RBAC setup and deploy locally build fdb operator.
+# TODO(johscheuer): Next setp is to add e2e tests for this setup.
+
+echo "For clean up run kind delete clusters ${cluster1} ${cluster2} ${cluster3} ${cluster4}"