Convert to kind for testing with multiple clusters

.travis.yml

@@ -20,7 +20,7 @@ install:
   -
 
 script:
-  - bash -x ./scripts/pre-commit.sh
+  - CONFIGURE_INSECURE_REGISTRY=y bash -x ./scripts/pre-commit.sh
   - vendor/github.com/kubernetes/repo-infra/verify/verify-boilerplate.sh -- rootdir="$GOPATH/src/github.com/kubernetes-sigs/federation-v2" -v
 
 after_success:

scripts/download-e2e-binaries.sh

@@ -21,11 +21,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# Use DEBUG=1 ./scripts/download-e2e-binaries.sh to get debug output
-[[ -z "${DEBUG:-""}" ]] || {
-  set -x
-}
-
 logEnd() {
   local msg='done.'
   [ "$1" -eq 0 ] || msg='Error downloading assets'

scripts/pre-commit.sh

@@ -23,6 +23,10 @@ set -o pipefail
 source "$(dirname "${BASH_SOURCE}")/util.sh"
 ROOT_DIR="$(cd "$(dirname "$0")/.." ; pwd)"
 MAKE_CMD="make -C ${ROOT_DIR}"
+NUM_CLUSTERS="${NUM_CLUSTERS:-2}"
+JOIN_CLUSTERS="${JOIN_CLUSTERS:-}"
+CONFIGURE_INSECURE_REGISTRY="${CONFIGURE_INSECURE_REGISTRY:-}"
+CONTAINER_REGISTRY_HOST="${CONTAINER_REGISTRY_HOST:-172.17.0.1:5000}"
 MANAGED_E2E_TEST_CMD="go test -v ./test/e2e -args -ginkgo.v -single-call-timeout=1m -ginkgo.trace -ginkgo.randomizeAllSpecs"
 # Specifying a kube config allows the tests to target deployed (unmanaged) fixture
 UNMANAGED_E2E_TEST_CMD="${MANAGED_E2E_TEST_CMD} -kubeconfig=${HOME}/.kube/config"
@@ -40,17 +44,113 @@ function run-e2e-tests-with-managed-fixture() {
   ${MANAGED_E2E_TEST_CMD}
 }
 
-function launch-minikube-cluster() {
-  # Move crictl to system path as expected by kubeadm.
-  sudo mv ${TEST_ASSET_PATH}/crictl /usr/bin/
-  sudo bash -c "export PATH=${PATH}; minikube start -p cluster1 --kubernetes-version v1.11.0 --vm-driver=none --v=4"
+docker_daemon_config="/etc/docker/daemon.json"
 
-  # Change ownership of .kube and .minikube config to use without sudo.
-  sudo chown -R $USER $HOME/.kube
-  sudo chgrp -R $USER $HOME/.kube
+function create-and-configure-insecure-registry() {
+  # Run insecure registry as container
+  docker run -d -p 5000:5000 --restart=always --name registry registry:2
 
-  sudo chown -R $USER $HOME/.minikube
-  sudo chgrp -R $USER $HOME/.minikube
+  if [[ -z "${CONFIGURE_INSECURE_REGISTRY}" ]]; then
+    return
+  fi
+
+  if sudo test -f "${docker_daemon_config}"; then
+    echo <<EOF "Error: ${docker_daemon_config} exists and \
+CONFIGURE_INSECURE_REGISTRY=${CONFIGURE_INSECURE_REGISTRY}. This script needs \
+to add an 'insecure-registries' entry with host '${CONTAINER_REGISTRY_HOST}' to \
+${docker_daemon_config}. Please make the necessary changes or backup and try again."
+EOF
+    return 1
+  fi
+
+  configure-insecure-registry-and-reload "sudo bash -c" $(pgrep dockerd)
+}
+
+function configure-insecure-registry-and-reload() {
+  local cmd_context="${1}" # context to run command e.g. sudo, docker exec
+  local docker_pid="${2}"
+  ${cmd_context} "$(insecure-registry-config-cmd)"
+  ${cmd_context} "$(reload-docker-daemon-cmd "${docker_pid}")"
+}
+
+function insecure-registry-config-cmd() {
+  echo "cat <<EOF > ${docker_daemon_config}
+{
+    \"insecure-registries\": [\"${CONTAINER_REGISTRY_HOST}\"]
+}
+EOF
+"
+}
+
+function reload-docker-daemon-cmd() {
+  echo "kill -s SIGHUP ${1}"
+}
+
+function create-clusters() {
+  local num_clusters=${1}
+
+  for i in $(seq ${num_clusters}); do
+    # kind will create cluster with name: kind-${i}
+    kind create cluster --name ${i}
+    # TODO(font): remove once all workarounds are addressed.
+    fixup-cluster ${i}
+  done
+
+  # TODO(font): kind will create separate kubeconfig files for each cluster.
+  # Remove once https://github.com/kubernetes-sigs/kind/issues/113 is resolved.
+  kubectl config view --flatten > ~/.kube/config
+  unset KUBECONFIG
+
+  echo "Waiting for clusters to be ready"
+  check-clusters-ready ${num_clusters}
+
+  # TODO(font): Configure insecure registry on kind host cluster. Remove once
+  # https://github.com/kubernetes-sigs/kind/issues/110 is resolved.
+  configure-insecure-registry-on-cluster 1
+
+  # Initialize list of clusters to join
+  join-cluster-list > /dev/null
+}
+
+function fixup-cluster() {
+  local i=${1} # cluster num
+
+  local kubeconfig_path="$(kind get kubeconfig-path --name ${i})"
+  export KUBECONFIG="${KUBECONFIG:-}:${kubeconfig_path}"
+
+  # Simplify context name
+  kubectl config rename-context kubernetes-admin@kind-${i} kind-${i}
+
+  # TODO(font): Need to set container IP address in order for clusters to reach
+  # kube API servers in other clusters until
+  # https://github.com/kubernetes-sigs/kind/issues/111 is resolved.
+  local container_ip_addr=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-${i}-control-plane)
+  sed -i "s/localhost/${container_ip_addr}/" ${kubeconfig_path}
+
+  # TODO(font): Need to rename auth user name to avoid conflicts when using
+  # multiple cluster kubeconfigs. Remove once
+  # https://github.com/kubernetes-sigs/kind/issues/112 is resolved.
+  sed -i "s/kubernetes-admin/kubernetes-kind-${i}-admin/" ${kubeconfig_path}
+}
+
+function check-clusters-ready() {
+  for i in $(seq ${1}); do
+    util::wait-for-condition 'ok' "kubectl --context kind-${i} get --raw=/healthz" 120
+  done
+}
+
+function configure-insecure-registry-on-cluster() {
+  configure-insecure-registry-and-reload "docker exec kind-${1}-control-plane bash -c" '$(pgrep dockerd)'
+}
+
+function join-cluster-list() {
+  if [[ -z "${JOIN_CLUSTERS}" ]]; then
+    for i in $(seq 2 ${NUM_CLUSTERS}); do
+      JOIN_CLUSTERS+="kind-${i} "
+    done
+    export JOIN_CLUSTERS=$(echo ${JOIN_CLUSTERS} | sed 's/ $//')
+  fi
+  echo "${JOIN_CLUSTERS}"
 }
 
 function run-e2e-tests-with-unmanaged-fixture() {
@@ -75,7 +175,7 @@ function check-install-yaml() {
   PATH="${PATH}:${base_dir}/bin" FEDERATION_NAMESPACE=federation-system \
     INSTALL_YAML=./hack/install-latest.yaml \
     ./scripts/generate-install-yaml.sh \
-    quay.io/kubernetes-multicluster/federation-v2:latest
+    ${CONTAINER_REGISTRY_HOST}/federation-v2:latest
   echo "Checking state of working tree after generating install yaml"
   check-git-state
 }
@@ -126,14 +226,14 @@ echo "Downloading e2e test dependencies"
 
 export PATH=${TEST_ASSET_PATH}:${PATH}
 
-echo "Launching minikube cluster"
-launch-minikube-cluster
+echo "Creating container registry on host"
+create-and-configure-insecure-registry
 
-echo "Waiting for minikube cluster to be ready"
-util::wait-for-condition 'ok' 'kubectl get --raw=/healthz' 120
+echo "Creating ${NUM_CLUSTERS} clusters"
+create-clusters ${NUM_CLUSTERS}
 
 echo "Deploying federation-v2"
-DOCKER_PUSH=false ./scripts/deploy-federation.sh quay.io/kubernetes-multicluster/federation-v2:e2e
+./scripts/deploy-federation.sh ${CONTAINER_REGISTRY_HOST}/federation-v2:e2e $(join-cluster-list)
 
 echo "Running go e2e tests with unmanaged fixture"
 run-e2e-tests-with-unmanaged-fixture
@@ -142,7 +242,7 @@ echo "Deleting federation-v2"
 ./scripts/delete-federation.sh
 
 echo "Deploying namespaced federation-v2"
-FEDERATION_NAMESPACE=foo NAMESPACED=y DOCKER_PUSH=false ./scripts/deploy-federation.sh quay.io/kubernetes-multicluster/federation-v2:e2e
+FEDERATION_NAMESPACE=foo NAMESPACED=y ./scripts/deploy-federation.sh ${CONTAINER_REGISTRY_HOST}/federation-v2:e2e $(join-cluster-list)
 
 echo "Running go e2e tests with unmanaged fixture"
 run-namespaced-e2e-tests-with-unmanaged-fixture