diff --git a/.travis.yml b/.travis.yml index 46eb0c55c3..dcad7da628 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - script: - - bash -x ./scripts/pre-commit.sh + - CONFIGURE_INSECURE_REGISTRY=y bash -x ./scripts/pre-commit.sh - vendor/github.com/kubernetes/repo-infra/verify/verify-boilerplate.sh -- rootdir="$GOPATH/src/github.com/kubernetes-sigs/federation-v2" -v after_success: diff --git a/scripts/download-e2e-binaries.sh b/scripts/download-e2e-binaries.sh index e0203e469d..d2b40a9f6d 100755 --- a/scripts/download-e2e-binaries.sh +++ b/scripts/download-e2e-binaries.sh @@ -21,11 +21,6 @@ set -o errexit set -o nounset set -o pipefail -# Use DEBUG=1 ./scripts/download-e2e-binaries.sh to get debug output -[[ -z "${DEBUG:-""}" ]] || { - set -x -} - logEnd() { local msg='done.' [ "$1" -eq 0 ] || msg='Error downloading assets' diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh index e60bc262aa..9ac9765747 100755 --- a/scripts/pre-commit.sh +++ b/scripts/pre-commit.sh @@ -23,6 +23,10 @@ set -o pipefail source "$(dirname "${BASH_SOURCE}")/util.sh" ROOT_DIR="$(cd "$(dirname "$0")/.." ; pwd)" MAKE_CMD="make -C ${ROOT_DIR}" +NUM_CLUSTERS="${NUM_CLUSTERS:-2}" +JOIN_CLUSTERS="${JOIN_CLUSTERS:-}" +CONFIGURE_INSECURE_REGISTRY="${CONFIGURE_INSECURE_REGISTRY:-}" +CONTAINER_REGISTRY_HOST="${CONTAINER_REGISTRY_HOST:-172.17.0.1:5000}" MANAGED_E2E_TEST_CMD="go test -v ./test/e2e -args -ginkgo.v -single-call-timeout=1m -ginkgo.trace -ginkgo.randomizeAllSpecs" # Specifying a kube config allows the tests to target deployed (unmanaged) fixture UNMANAGED_E2E_TEST_CMD="${MANAGED_E2E_TEST_CMD} -kubeconfig=${HOME}/.kube/config" @@ -40,17 +44,113 @@ function run-e2e-tests-with-managed-fixture() { ${MANAGED_E2E_TEST_CMD} } -function launch-minikube-cluster() { - # Move crictl to system path as expected by kubeadm. - sudo mv ${TEST_ASSET_PATH}/crictl /usr/bin/ - sudo bash -c "export PATH=${PATH}; minikube start -p cluster1 --kubernetes-version v1.11.0 --vm-driver=none --v=4" +docker_daemon_config="/etc/docker/daemon.json" - # Change ownership of .kube and .minikube config to use without sudo. - sudo chown -R $USER $HOME/.kube - sudo chgrp -R $USER $HOME/.kube +function create-and-configure-insecure-registry() { + # Run insecure registry as container + docker run -d -p 5000:5000 --restart=always --name registry registry:2 - sudo chown -R $USER $HOME/.minikube - sudo chgrp -R $USER $HOME/.minikube + if [[ -z "${CONFIGURE_INSECURE_REGISTRY}" ]]; then + return + fi + + if sudo test -f "${docker_daemon_config}"; then + echo < ${docker_daemon_config} +{ + \"insecure-registries\": [\"${CONTAINER_REGISTRY_HOST}\"] +} +EOF +" +} + +function reload-docker-daemon-cmd() { + echo "kill -s SIGHUP ${1}" +} + +function create-clusters() { + local num_clusters=${1} + + for i in $(seq ${num_clusters}); do + # kind will create cluster with name: kind-${i} + kind create cluster --name ${i} + # TODO(font): remove once all workarounds are addressed. + fixup-cluster ${i} + done + + # TODO(font): kind will create separate kubeconfig files for each cluster. + # Remove once https://github.com/kubernetes-sigs/kind/issues/113 is resolved. + kubectl config view --flatten > ~/.kube/config + unset KUBECONFIG + + echo "Waiting for clusters to be ready" + check-clusters-ready ${num_clusters} + + # TODO(font): Configure insecure registry on kind host cluster. Remove once + # https://github.com/kubernetes-sigs/kind/issues/110 is resolved. + configure-insecure-registry-on-cluster 1 + + # Initialize list of clusters to join + join-cluster-list > /dev/null +} + +function fixup-cluster() { + local i=${1} # cluster num + + local kubeconfig_path="$(kind get kubeconfig-path --name ${i})" + export KUBECONFIG="${KUBECONFIG:-}:${kubeconfig_path}" + + # Simplify context name + kubectl config rename-context kubernetes-admin@kind-${i} kind-${i} + + # TODO(font): Need to set container IP address in order for clusters to reach + # kube API servers in other clusters until + # https://github.com/kubernetes-sigs/kind/issues/111 is resolved. + local container_ip_addr=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-${i}-control-plane) + sed -i "s/localhost/${container_ip_addr}/" ${kubeconfig_path} + + # TODO(font): Need to rename auth user name to avoid conflicts when using + # multiple cluster kubeconfigs. Remove once + # https://github.com/kubernetes-sigs/kind/issues/112 is resolved. + sed -i "s/kubernetes-admin/kubernetes-kind-${i}-admin/" ${kubeconfig_path} +} + +function check-clusters-ready() { + for i in $(seq ${1}); do + util::wait-for-condition 'ok' "kubectl --context kind-${i} get --raw=/healthz" 120 + done +} + +function configure-insecure-registry-on-cluster() { + configure-insecure-registry-and-reload "docker exec kind-${1}-control-plane bash -c" '$(pgrep dockerd)' +} + +function join-cluster-list() { + if [[ -z "${JOIN_CLUSTERS}" ]]; then + for i in $(seq 2 ${NUM_CLUSTERS}); do + JOIN_CLUSTERS+="kind-${i} " + done + export JOIN_CLUSTERS=$(echo ${JOIN_CLUSTERS} | sed 's/ $//') + fi + echo "${JOIN_CLUSTERS}" } function run-e2e-tests-with-unmanaged-fixture() { @@ -75,7 +175,7 @@ function check-install-yaml() { PATH="${PATH}:${base_dir}/bin" FEDERATION_NAMESPACE=federation-system \ INSTALL_YAML=./hack/install-latest.yaml \ ./scripts/generate-install-yaml.sh \ - quay.io/kubernetes-multicluster/federation-v2:latest + ${CONTAINER_REGISTRY_HOST}/federation-v2:latest echo "Checking state of working tree after generating install yaml" check-git-state } @@ -126,14 +226,14 @@ echo "Downloading e2e test dependencies" export PATH=${TEST_ASSET_PATH}:${PATH} -echo "Launching minikube cluster" -launch-minikube-cluster +echo "Creating container registry on host" +create-and-configure-insecure-registry -echo "Waiting for minikube cluster to be ready" -util::wait-for-condition 'ok' 'kubectl get --raw=/healthz' 120 +echo "Creating ${NUM_CLUSTERS} clusters" +create-clusters ${NUM_CLUSTERS} echo "Deploying federation-v2" -DOCKER_PUSH=false ./scripts/deploy-federation.sh quay.io/kubernetes-multicluster/federation-v2:e2e +./scripts/deploy-federation.sh ${CONTAINER_REGISTRY_HOST}/federation-v2:e2e $(join-cluster-list) echo "Running go e2e tests with unmanaged fixture" run-e2e-tests-with-unmanaged-fixture @@ -142,7 +242,7 @@ echo "Deleting federation-v2" ./scripts/delete-federation.sh echo "Deploying namespaced federation-v2" -FEDERATION_NAMESPACE=foo NAMESPACED=y DOCKER_PUSH=false ./scripts/deploy-federation.sh quay.io/kubernetes-multicluster/federation-v2:e2e +FEDERATION_NAMESPACE=foo NAMESPACED=y ./scripts/deploy-federation.sh ${CONTAINER_REGISTRY_HOST}/federation-v2:e2e $(join-cluster-list) echo "Running go e2e tests with unmanaged fixture" run-namespaced-e2e-tests-with-unmanaged-fixture