Add E2E test for antrea multiclusters (antrea-io#3124)

Signed-off-by: hjiajing <[email protected]>
luolanzone · Jan 16, 2022 · e4ea620 · e4ea620
1 parent 5f32674
commit e4ea620
Show file tree

Hide file tree

Showing 14 changed files with 1,154 additions and 1 deletion.
diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh
@@ -0,0 +1,293 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+function echoerr {
+    >&2 echo "$@"
+}
+
+DEFAULT_WORKDIR="/var/lib/jenkins"
+DEFAULT_KUBECONFIG_PATH=$DEFAULT_WORKDIR/kube.conf
+WORKDIR=$DEFAULT_WORKDIR
+TESTCASE=""
+TEST_FAILURE=false
+DOCKER_REGISTRY=$(head -n1 "${WORKSPACE}/ci/docker-registry")
+GO_VERSION=$(head -n1 "${WORKSPACE}/build/images/deps/go-version")
+IMAGE_PULL_POLICY="Always"
+MULTICLUSTER_KUBECONFIG_PATH=$WORKDIR/.kube
+LEADER_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/leader"
+EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east"
+WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west"
+
+CONTROL_PLANE_NODE_ROLE="control-plane,master"
+
+multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
+membercluter_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
+
+
+CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=48h"
+
+_usage="Usage: $0 [--kubeconfigs-path <KubeconfigSavePath>] [--workdir <HomePath>]
+                  [--testcase <e2e>]
+
+Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set.
+
+        --kubeconfigs-path            Path of cluster set kubeconfigs.
+        --workdir                     Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
+        --testcase                    Antrea multi-cluster e2e test cases on a Linux cluster set.
+        --registry                    The docker registry to use instead of dockerhub."
+
+function print_usage {
+    echoerr "$_usage"
+}
+
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    --kubeconfigs-path)
+    MULTICLUSTER_KUBECONFIG_PATH="$2"
+    shift 2
+    ;;
+    --workdir)
+    WORKDIR="$2"
+    shift 2
+    ;;
+    --testcase)
+    TESTCASE="$2"
+    shift 2
+    ;;
+    --registry)
+    DOCKER_REGISTRY="$2"
+    shift 2
+    ;;
+    -h|--help)
+    print_usage
+    exit 0
+    ;;
+    *)    # unknown option
+    echoerr "Unknown option $1"
+    exit 1
+    ;;
+esac
+done
+
+function clean_tmp() {
+    echo "===== Clean up stale files & folders older than 7 days under /tmp ====="
+    CLEAN_LIST=(
+        "*codecov*"
+        "kustomize-*"
+        "*antrea*"
+        "go-build*"
+    )
+    for item in "${CLEAN_LIST[@]}"; do
+        find /tmp -name "${item}" -mtime +7 -exec rm -rf {} \; 2>&1 | grep -v "Permission denied" || true
+    done
+    find ${WORKDIR} -name "support-bundles*" -mtime +7 -exec rm -rf {} \; 2>&1 | grep -v "Permission denied" || true
+}
+
+
+function cleanup_multicluster_ns {
+    ns=$1
+    kubeconfig=$2
+
+    kubectl delete ns "${ns}" --ignore-not-found=true ${kubeconfig} --timeout=30s || true
+}
+
+function cleanup_multicluster_controller {
+    echo "====== Cleanup Multicluster Controller Installation ======"
+    kubeconfig=$1
+    for multicluster_yml in ${WORKSPACE}/multicluster/test/yamls/*.yml; do
+        kubectl delete -f $multicluster_yml $kubeconfig --ignore-not-found=true  --timeout=30s || true
+    done
+
+    for multicluster_yml in ${WORKSPACE}/multicluster/build/yamls/*.yml; do
+        kubectl delete -f $multicluster_yml $kubeconfig --ignore-not-found=true --timeout=30s || true
+    done
+}
+
+function cleanup_multicluster_antrea {
+    echo "====== Cleanup Antrea controller and agent ======"
+    kubeconfig=$1
+    kubectl get pod -n kube-system -l component=antrea-agent --no-headers=true $kubeconfig | awk '{print $1}' | while read AGENTNAME; do
+       kubectl exec $AGENTNAME -c antrea-agent -n kube-system ${kubeconfig}  ovs-vsctl del-port br-int gw0 || true
+    done
+
+   for antrea_yml in ${WORKDIR}/*.yml; do
+        kubectl delete -f $antrea_yml --ignore-not-found=true ${kubeconfig} --timeout=30s || true
+    done
+}
+
+function clean_multicluster {
+    echo "====== Cleanup Multicluster Antrea Installation in clusters ======"
+    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    do
+        cleanup_multicluster_ns "antrea-multicluster-test" $kubeconfig
+        cleanup_multicluster_ns "antrea-mcs-ns" $kubeconfig
+        cleanup_multicluster_controller $kubeconfig
+        cleanup_multicluster_antrea $kubeconfig
+    done
+}
+
+function wait_for_antrea_multicluster_pods_ready {
+    kubeconfig=$1
+    kubectl apply -f build/yamls/antrea.yml "${kubeconfig}"
+    kubectl rollout restart deployment/coredns -n kube-system "${kubeconfig}"
+    kubectl rollout status deployment/coredns -n kube-system "${kubeconfig}"
+    kubectl rollout status deployment.apps/antrea-controller -n kube-system "${kubeconfig}"
+    kubectl rollout status daemonset/antrea-agent -n kube-system "${kubeconfig}"
+}
+
+function wait_for_multicluster_controller_ready {
+    kubectl create ns antrea-mcs-ns  "${LEADER_CLUSTER_CONFIG}" || true
+    kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
+    kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}"
+    kubectl rollout status deployment/antrea-mc-controller -n antrea-mcs-ns "${LEADER_CLUSTER_CONFIG}" || true
+    kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
+    kubectl create -f ./multicluster/test/yamls/leader-access-token-secret.yml "${LEADER_CLUSTER_CONFIG}" || true
+    kubectl get secret -n antrea-mcs-ns leader-access-token "${LEADER_CLUSTER_CONFIG}" -o yaml > ./multicluster/test/yamls/leader-access-token.yml
+
+    sed -i '/uid:/d' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i '/resourceVersion/d' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i '/last-applied-configuration/d' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i '/type/d' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i '/creationTimestamp/d' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i 's/antrea-multicluster-member-access-sa/antrea-multicluster-controller/g' ./multicluster/test/yamls/leader-access-token.yml
+    sed -i 's/antrea-mcs-ns/kube-system/g' ./multicluster/test/yamls/leader-access-token.yml
+    echo "type: Opaque" >>./multicluster/test/yamls/leader-access-token.yml
+
+    for config in ${membercluter_kubeconfigs[@]};
+    do
+        kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config}
+        kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config}
+        kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml ${config}
+    done
+
+    kubectl apply -f ./multicluster/test/yamls/east-member-cluster.yml "${EAST_CLUSTER_CONFIG}"
+    kubectl apply -f ./multicluster/test/yamls/west-member-cluster.yml "${WEST_CLUSTER_CONFIG}"
+    kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}"
+}
+
+function deliver_antrea_multicluster {
+    echo "====== Building Antrea for the Following Commit ======"
+    export GO111MODULE=on
+    export GOPATH=${WORKDIR}/go
+    export GOROOT=/usr/local/go
+    export PATH=${GOROOT}/bin:$PATH
+
+    git show --numstat
+    make clean
+    ${CLEAN_STALE_IMAGES}
+
+    cp -f build/yamls/*.yml $WORKDIR
+    docker pull $DOCKER_REGISTRY/antrea/antrea-ubuntu:latest
+    echo "====== Delivering Antrea to all the Nodes ======"
+    docker save -o ${WORKDIR}/antrea-ubuntu.tar $DOCKER_REGISTRY/antrea/antrea-ubuntu:latest
+
+
+    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    do
+       kubectl get nodes -o wide --no-headers=true ${kubeconfig}| awk '{print $6}' | while read IP; do
+            rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar
+            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true
+       done
+    done
+}
+
+function deliver_multicluster_controller {
+    echo "====== Build Antrea Multiple Cluster Controller and YAMLs ======"
+    export GO111MODULE=on
+    export GOPATH=${WORKDIR}/go
+    export GOROOT=/usr/local/go
+    export PATH=${GOROOT}/bin:$PATH
+
+    docker images | grep 'mc-controller' | awk '{print $3}' | xargs -r docker rmi || true
+    cd multicluster && mkdir -p bin && ln -s /usr/local/bin/kustomize ./bin/kustomize || true
+    make docker-build && cd ..
+
+    docker save antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar
+    ./multicluster/hack/generate-manifest.sh -l antrea-mcs-ns >./multicluster/test/yamls/manifest.yml
+
+    for kubeconfig in ${multicluster_kubeconfigs[@]}
+    do
+        kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do
+            rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar
+            ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true
+        done
+    done
+
+    leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
+    sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml
+    sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml
+
+    for kubeconfig in ${membercluter_kubeconfigs[@]}
+    do
+       ip=$(kubectl get nodes -o wide --no-headers=true ${EAST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
+       rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@[${ip}]:${WORKDIR}/serviceexport.yml
+    done
+}
+
+function run_multicluster_e2e {
+   echo "====== Running Multicluster e2e Tests ======"
+    export GO111MODULE=on
+    export GOPATH=${WORKDIR}/go
+    export GOROOT=/usr/local/go
+    export GOCACHE=${WORKDIR}/.cache/go-build
+    export PATH=$GOROOT/bin:$PATH
+
+    wait_for_antrea_multicluster_pods_ready "${LEADER_CLUSTER_CONFIG}"
+    wait_for_antrea_multicluster_pods_ready "${EAST_CLUSTER_CONFIG}"
+    wait_for_antrea_multicluster_pods_ready "${WEST_CLUSTER_CONFIG}"
+
+    wait_for_multicluster_controller_ready
+
+    docker pull "${DOCKER_REGISTRY}/antrea/nginx"
+    docker tag "${DOCKER_REGISTRY}/antrea/nginx:latest" "nginx:latest"
+    docker save nginx:latest -o "${WORKDIR}"/nginx.tar
+
+    for kubeconfig in ${membercluter_kubeconfigs[@]}
+    do
+        kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do
+            rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/nginx.tar jenkins@["${IP}"]:"${WORKDIR}"/nginx.tar
+            ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/nginx.tar" || true
+        done
+
+    done
+
+    set +e
+    mkdir -p `pwd`/antrea-multicluster-test-logs
+    go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir  `pwd`/antrea-multicluster-test-logs
+    if [[ "$?" != "0" ]]; then
+        TEST_FAILURE=true
+    fi
+    set -e
+}
+
+trap clean_multicluster EXIT
+clean_tmp
+
+if [[ ${TESTCASE} =~ "e2e" ]]; then
+    deliver_antrea_multicluster
+    deliver_multicluster_controller
+    run_multicluster_e2e
+fi
+
+if [[ ${TEST_FAILURE} == true ]]; then
+    exit 1
+fi
diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -52,7 +52,7 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
 
         --kubeconfig             Path of cluster kubeconfig.
         --workdir                Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
-        --testcase               Windows install OVS, Conformance and Network Policy or Antrea e2e testcases on a Windows or Linux cluster. It can also be flexible ipam e2e test.
+        --testcase               Windows install OVS, Conformance and Network Policy or Antrea e2e test cases on a Windows or Linux cluster. It can also be flexible ipam e2e test.
         --registry               The docker registry to use instead of dockerhub.
         --proxyall               Enable proxyAll to test AntreaProxy.
         --flexible-ipam          Run tests in flexible ipam mode"

diff --git a/multicluster/test/e2e/README.md b/multicluster/test/e2e/README.md
@@ -0,0 +1,13 @@
+# Running the Antrea Multicluster end-to-end tests
+
+## Creating the test Kubernetes ClusterSet
+
+The tests must be run on an actual Kubernetes cluster set, and there must be at least three clusters. We can create the clusters using [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm) or following [the e2e test instructions](https://github.com/antrea-io/antrea/blob/main/test/e2e/README.md) or through other ways.
+
+The three clusters are called leader cluster, west cluster and east cluster. Pod CIDR of each cluster should not be the same. For example, Pod CIDRs could be `192.168.0.1/22`,`192.168.4.1/22` and `192.168.8.1/22` in three clusters.
+
+## Running the tests
+
+Make sure that your clusters are provisioned and that the Antrea build artifacts are pushed to all the Nodes. If you install the multicluster manually, you can then run the tests from the top-level directory with `go test -v antrea.io/antrea/multicluster/test/e2e` or you can just run `bash ci/jenkins/test-mc.sh --testcase e2e`.
+
+When you use `test-mc.sh`, make sure your kubeconfig files of the clusters in the specific path of the script, or you can change the parameter `MULTICLUSTER_KUBECONFIG_PATH` and other parameters to your own path in the script.