Add multi-cluster kind setup for testing

Makefile

@@ -39,6 +39,7 @@ GORELEASER_PKG=github.com/goreleaser/[email protected]
 GORELEASER=$(GOBIN)/goreleaser
 BUILD_DEPS?=
 BUILDER?="docker"
+BUILDER_ARGS?=
 KUBECTL_ARGS?=
 
 define godep
@@ -174,7 +175,7 @@ ${GENERATED_GO}: ${GO_SRC} hack/boilerplate.go.txt ${CONTROLLER_GEN}
 
 # Build the container image
 container-build: test_if_changed
-	$(BUILDER) build --build-arg=TAG=${TAG} ${img_build_args} -t ${IMG} .
+	$(BUILDER) build --build-arg=TAG=${TAG} ${img_build_args} $(BUILDER_ARGS) -t ${IMG} .
 
 # Push the container image
 container-push:

config/development/kustomization.yaml.sample

@@ -3,12 +3,14 @@ kind: Kustomization
 
 patchesStrategicMerge:
 - test_certs.yaml
-- backup_credentials.yaml
+# - backup_credentials.yaml
 - debug_logs.yaml
 
 resources:
 - ../deployment
-- ../minio
+# Disable MinIO, the old apiextensions.k8s.io/v1beta1 is not served anymore since 1.22
+# see: https://kubernetes.io/docs/reference/using-api/deprecation-guide/#customresourcedefinition-v122
+# - ../minio
 
 images:
 - name: foundationdb/fdb-kubernetes-operator

config/minio/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 patchesStrategicMerge:
-- instance_overrides.yaml
+- instance_overrides.yaml

config/test-certs/generate_secrets.bash

@@ -1,4 +1,4 @@
-#! /bin/bash
+#!/usr/bin/env bash
 
 set -eu
 
@@ -23,6 +23,9 @@ set -eu
 
 # This script generates secrets with test certs for use in local testing.
 
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+cd "${SCRIPT_DIR}"
+
 kubectl delete secrets -l app=fdb-kubernetes-operator
-kubectl create secret tls fdb-kubernetes-operator-secrets --key=config/test-certs/key.pem --cert=config/test-certs/cert.pem
+kubectl create secret tls fdb-kubernetes-operator-secrets --key=${SCRIPT_DIR}/key.pem --cert=${SCRIPT_DIR}/cert.pem
 kubectl label secret fdb-kubernetes-operator-secrets app=fdb-kubernetes-operator

config/tests/multi_dc/Readme.md

@@ -0,0 +1,20 @@
+# Multi DC example
+
+Ensure that you ran the `setup_e2e.sh` script and setup the environment variables for the clusters.
+This setup is currently meant to run manually. In the future this will be rewritten to be a real e2e test.
+
+## Create the multi-dc cluster
+
+This will bringup a FDB cluster running across 3 different Kubernetes clusters in a multi-region configuration.
+
+```bash
+./create.bash
+```
+
+## Delete
+
+This will remove all created resources:
+
+```bash
+./bash.bash
+```

config/tests/multi_dc/create.bash

@@ -1,4 +1,6 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
+set -eu
 
 # This directory provides an example of creating a cluster using the multi-KC
 # replication topology.
@@ -15,14 +17,24 @@ DIR="${BASH_SOURCE%/*}"
 
 . $DIR/functions.bash
 
-applyFile $DIR/stage_1.yaml dc1 '""'
-checkReconciliationLoop test-cluster-dc1
-connectionString=$(getConnectionString test-cluster-dc1)
+# To test a multi-region FDB cluster setup we need to have 3 Kubernetes clusters
+cluster1=${CLUSTER1:-cluster1}
+cluster2=${CLUSTER2:-cluster2}
+cluster3=${CLUSTER3:-cluster3}
+
+# For all the clusters we have to create tha according kubeconfig
+kubeconfig1=${KUBECONFIG1:-"${cluster1}.kubeconfig"}
+kubeconfig2=${KUBECONFIG2:-"${cluster2}.kubeconfig"}
+kubeconfig3=${KUBECONFIG3:-"${cluster3}.kubeconfig"}
+
+applyFile "${DIR}/stage_1.yaml" dc1 '""' "${kubeconfig1}"
+checkReconciliationLoop test-cluster-dc1 "${kubeconfig1}"
+connectionString=$(getConnectionString test-cluster-dc1 "${kubeconfig1}")
 
-applyFile $DIR/final.yaml dc1 $connectionString
-applyFile $DIR/final.yaml dc2 $connectionString
-applyFile $DIR/final.yaml dc3 $connectionString
+applyFile "${DIR}/final.yaml" dc1 "${connectionString}" "${kubeconfig1}"
+applyFile "${DIR}/final.yaml" dc2 "${connectionString}" "${kubeconfig2}"
+applyFile "${DIR}/final.yaml" dc3 "${connectionString}" "${kubeconfig3}"
 
-checkReconciliationLoop test-cluster-dc1
-checkReconciliationLoop test-cluster-dc2
-checkReconciliationLoop test-cluster-dc3
+checkReconciliationLoop test-cluster-dc1 "${kubeconfig1}"
+checkReconciliationLoop test-cluster-dc2 "${kubeconfig2}"
+checkReconciliationLoop test-cluster-dc3 "${kubeconfig3}"

config/tests/multi_dc/delete.bash

@@ -1,3 +1,17 @@
-#! /bin/bash
+#!/usr/bin/env bash
 
-kubectl delete fdb -l cluster-group=test-cluster
+set -eu
+
+# To test a multi-region FDB cluster setup we need to have 3 Kubernetes clusters
+cluster1=${CLUSTER1:-cluster1}
+cluster2=${CLUSTER2:-cluster2}
+cluster3=${CLUSTER3:-cluster3}
+
+# For all the clusters we have to create tha according kubeconfig
+kubeconfig1=${KUBECONFIG1:-"${cluster1}.kubeconfig"}
+kubeconfig2=${KUBECONFIG2:-"${cluster2}.kubeconfig"}
+kubeconfig3=${KUBECONFIG3:-"${cluster3}.kubeconfig"}
+
+kubectl --kubeconfig "${kubeconfig1}" delete fdb -l cluster-group=test-cluster
+kubectl --kubeconfig "${kubeconfig2}" delete fdb -l cluster-group=test-cluster
+kubectl --kubeconfig "${kubeconfig3}" delete fdb -l cluster-group=test-cluster

config/tests/multi_dc/final.yaml

@@ -7,21 +7,18 @@ apiVersion: apps.foundationdb.org/v1beta2
 kind: FoundationDBCluster
 metadata:
   labels:
-    controller-tools.k8s.io: "1.0"
     cluster-group: test-cluster
   name: test-cluster-$dc
 spec:
-  version: 6.2.30
+  version: 7.1.25
   faultDomain:
     key: foundationdb.org/none
-  instanceIDPrefix: $dc
+  processGroupIDPrefix: $dc
   dataCenter: $dc
   processCounts:
     stateless: -1
     log: $logCount
   seedConnectionString: $connectionString
-  customParameters:
-    - "knob_disable_posix_kernel_aio=1"
   databaseConfiguration:
     redundancy_mode: "double"
     usable_regions: 2
@@ -48,6 +45,8 @@ spec:
         satellite_logs: 3
   processes:
     general:
+      customParameters:
+      - "knob_disable_posix_kernel_aio=1"
       volumeClaimTemplate:
         spec:
           resources:

config/tests/multi_dc/functions.bash

@@ -1,22 +1,21 @@
 function applyFile() {
-	path=$1
-	dc=$2
-	connectionString=$3
+	dc=${2}
+
 	if [[ "${dc}" == "dc3" ]]; then
 		logCount=0
 	else
 		logCount=-1
 	fi
-	
-  sed -e "s/\$dc/$dc/" "${path}" | sed -e "s/\$logCount/$logCount/" | sed -e "s/\$connectionString/$connectionString/" | kubectl apply -f -
+
+	dc="${dc}" connectionString="${3}" logCount="${logCount}" envsubst < "${1}"| kubectl --kubeconfig "${4}" apply -f -
 }
 
 function checkReconciliation() {
 	clusterName=$1
 
-	generationsOutput=$(kubectl get fdb "${clusterName}" -o jsonpath='{.metadata.generation} {.status.generations.reconciled}')
+	generationsOutput=$(kubectl --kubeconfig "${2}" get fdb "${clusterName}" -o jsonpath='{.metadata.generation} {.status.generations.reconciled}')
 	read -ra generations <<< "${generationsOutput}"
-	if [[ ("${generations[1]}" != "")  && ("${generations[0]}" == "${generations[1]}") ]]; then
+	if [[ ("${#generations[@]}" -ge 2) && ("${generations[0]}" == "${generations[1]}") ]]; then
 		return 1
 	else
 		echo "Latest generations for $clusterName: $generationsOutput"
@@ -25,20 +24,12 @@ function checkReconciliation() {
 }
 
 function getConnectionString() {
-	clusterName=$1
-
-	kubectl get fdb "${clusterName}" -o jsonpath='{.status.connectionString}'
+	kubectl --kubeconfig "${2}" get fdb "${1}" -o jsonpath='{.status.connectionString}'
 }
 
 function checkReconciliationLoop() {
-	reconciled=0
-	name=$1
-	while [ $reconciled -ne 1 ] ; do
-		checkReconciliation "${name}"
-		reconciled=$?
-		if [[ $reconciled -ne 1 ]]; then
-			echo "Waiting for reconciliation"
-			sleep 5
-		fi
+	while checkReconciliation "${1}" "${2}"; do
+		echo "Waiting for reconciliation"
+		sleep 5
 	done
 }

config/tests/multi_dc/stage_1.yaml

@@ -10,17 +10,15 @@ metadata:
     cluster-group: test-cluster
   name: test-cluster-$dc
 spec:
-  version: 6.2.30
+  version: 7.1.25
   faultDomain:
     key: foundationdb.org/none
-  instanceIDPrefix: $dc
+  processGroupIDPrefix: $dc
   dataCenter: $dc
   processCounts:
     stateless: -1
     log: $logCount
   seedConnectionString: $connectionString
-  customParameters:
-    - "knob_disable_posix_kernel_aio=1"
   databaseConfiguration:
     redundancy_mode: "double"
     usable_regions: 1
@@ -30,6 +28,8 @@ spec:
             priority: 1
   processes:
     general:
+      customParameters:
+      - "knob_disable_posix_kernel_aio=1"
       volumeClaimTemplate:
         spec:
           resources:

config/tests/multi_kc/create.bash

@@ -1,4 +1,6 @@
-#! /bin/bash
+#!/usr/bin/env bash
+
+set -eu
 
 # This directory provides an example of creating a cluster using the multi-KC
 # replication topology.
@@ -15,14 +17,24 @@ DIR="${BASH_SOURCE%/*}"
 
 . $DIR/functions.bash
 
-applyFile $DIR/stage_1.yaml dc1 '""'
-checkReconciliationLoop test-cluster-dc1
-connectionString=$(getConnectionString test-cluster-dc1)
+# To test a multi-region FDB cluster setup we need to have 3 Kubernetes clusters
+cluster1=${CLUSTER1:-cluster1}
+cluster2=${CLUSTER2:-cluster2}
+cluster3=${CLUSTER3:-cluster3}
+
+# For all the clusters we have to create tha according kubeconfig
+kubeconfig1=${KUBECONFIG1:-"${cluster1}.kubeconfig"}
+kubeconfig2=${KUBECONFIG2:-"${cluster2}.kubeconfig"}
+kubeconfig3=${KUBECONFIG3:-"${cluster3}.kubeconfig"}
+
+applyFile $DIR/stage_1.yaml dc1 '""' "${kubeconfig1}"
+checkReconciliationLoop test-cluster-dc1 "${kubeconfig1}"
+connectionString=$(getConnectionString test-cluster-dc1 "${kubeconfig1}")
 
-applyFile $DIR/final.yaml dc1 $connectionString
-applyFile $DIR/final.yaml dc2 $connectionString
-applyFile $DIR/final.yaml dc3 $connectionString
+applyFile $DIR/final.yaml dc1 $connectionString "${kubeconfig1}"
+applyFile $DIR/final.yaml dc2 $connectionString "${kubeconfig2}"
+applyFile $DIR/final.yaml dc3 $connectionString "${kubeconfig3}"
 
-checkReconciliationLoop test-cluster-dc1
-checkReconciliationLoop test-cluster-dc2
-checkReconciliationLoop test-cluster-dc3
+checkReconciliationLoop test-cluster-dc1 "${kubeconfig1}"
+checkReconciliationLoop test-cluster-dc2 "${kubeconfig2}"
+checkReconciliationLoop test-cluster-dc3 "${kubeconfig3}"

config/tests/multi_kc/delete.bash

@@ -1,3 +1,17 @@
-#! /bin/bash
+#!/usr/bine/env bash
 
-kubectl delete fdb -l cluster-group=test-cluster
+set -eu
+
+# To test a multi-region FDB cluster setup we need to have 3 Kubernetes clusters
+cluster1=${CLUSTER1:-cluster1}
+cluster2=${CLUSTER2:-cluster2}
+cluster3=${CLUSTER3:-cluster3}
+
+# For all the clusters we have to create tha according kubeconfig
+kubeconfig1=${KUBECONFIG1:-"${cluster1}.kubeconfig"}
+kubeconfig2=${KUBECONFIG2:-"${cluster2}.kubeconfig"}
+kubeconfig3=${KUBECONFIG3:-"${cluster3}.kubeconfig"}
+
+kubectl --kubeconfig "${kubeconfig1}" delete fdb -l cluster-group=test-cluster
+kubectl --kubeconfig "${kubeconfig2}" delete fdb -l cluster-group=test-cluster
+kubectl --kubeconfig "${kubeconfig3}" delete fdb -l cluster-group=test-cluster