Skip to content

Commit

Permalink
Add E2E test for antrea multiclusters
Browse files Browse the repository at this point in the history
  • Loading branch information
hjiajing committed Dec 12, 2021
1 parent f55dbad commit c1a7bfb
Show file tree
Hide file tree
Showing 12 changed files with 985 additions and 2 deletions.
223 changes: 221 additions & 2 deletions ci/jenkins/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ GO_VERSION=$(head -n1 "${WORKSPACE}/build/images/deps/go-version")
IMAGE_PULL_POLICY="Always"
PROXY_ALL=false
FLEXIBLE_IPAM=false
MULTICLUSTER_KUBECONFIG_PATH=$WORKDIR/.kube
LEADER_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/leader"
EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east"
WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west"

WINDOWS_CONFORMANCE_FOCUS="\[sig-network\].+\[Conformance\]|\[sig-windows\]"
WINDOWS_CONFORMANCE_SKIP="\[LinuxOnly\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]|\[Privileged\]|should be able to change the type from|\[sig-network\] Services should be able to create a functioning NodePort service \[Conformance\]|Service endpoints latency should not be very high|should be able to create a functioning NodePort service for Windows"
Expand All @@ -41,7 +45,10 @@ CONFORMANCE_SKIP="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[
NETWORKPOLICY_SKIP="should allow egress access to server in CIDR block|should enforce except clause while egress access to server in CIDR block"

# TODO: change to "control-plane" when testbeds are updated to K8s v1.20
CONTROL_PLANE_NODE_ROLE="master"
CONTROL_PLANE_NODE_ROLE="control-plane,master"

multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)


CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=48h"

Expand Down Expand Up @@ -141,6 +148,53 @@ function clean_antrea {
done
}

function cleanup_multicluster_ns {
ns=$1
kubeconfig=$2

kubectl get pod -n "${ns}" --no-headers=true ${kubeconfig}| awk '{print $1}' | while read pod_name; do
kubectl delete pod "${pod_name}" -n "${ns}" --force --grace-period 0 ${kubeconfig} --timeout=30s
done
kubectl get svc -n "${ns}" --no-headers=true ${kubeconfig}| awk '{print $1}' | while read svc_name; do
kubectl delete svc "${svc_name}" -n "${ns}" --force --grace-period 0 ${kubeconfig} --timeout=30s
done
kubectl delete ns "${ns}" --ignore-not-found=true ${kubeconfig} --timeout=30s || true
}

function cleanup_multicluster_controller {
echo "====== Cleanup Multicluster Controller Installation ======"
kubeconfig=$1
for multicluster_yml in ./multicluster/test/yamls/*.yml; do
kubectl delete -f $multicluster_yml $kubeconfig --ignore-not-found=true --timeout=30s || true
done

for multicluster_yml in ./multicluster/build/yamls/*.yml; do
kubectl delete -f $multicluster_yml $kubeconfig --ignore-not-found=true --timeout=30s || true
done
}

function cleanup_multicluster_antrea {
echo "====== Cleanup Multicluster Antrea controller and agent ======"
kubeconfig=$1
kubectl get pod -n kube-system -l component=antrea-agent --no-headers=true $kubeconfig | awk '{print $1}' | while read AGENTNAME; do
kubectl exec $AGENTNAME -c antrea-agent -n kube-system ${kubeconfig} ovs-vsctl del-port br-int gw0 || true
done

for antrea_yml in ${WORKDIR}/*.yml; do
kubectl delete -f $antrea_yml --ignore-not-found=true ${kubeconfig} --timeout=30s || true
done
}

function clean_multicluster {
echo "====== Cleanup Multicluster Antrea Installation in clusters ======"
for kubeconfig in ${multicluster_kubeconfigs[@]}
do
cleanup_multicluster_ns "antrea-multicluster-test" $kubeconfig
cleanup_multicluster_controller $kubeconfig
cleanup_multicluster_antrea $kubeconfig
done
}

function clean_for_windows_install_cni {
# https://github.com/antrea-io/antrea/issues/1577
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role && $1 ~ /win/ {print $6}' | while read IP; do
Expand Down Expand Up @@ -207,6 +261,46 @@ function collect_windows_network_info_and_logs {
tar zcf debug_logs.tar.gz "${DEBUG_LOG_PATH}"
}

function wait_for_antrea_multicluster_pods_ready {
kubeconfig=$1
kubectl apply -f build/yamls/antrea.yml "${kubeconfig}"
kubectl rollout restart deployment/coredns -n kube-system "${kubeconfig}"
kubectl rollout status deployment/coredns -n kube-system "${kubeconfig}"
kubectl rollout status deployment.apps/antrea-controller -n kube-system "${kubeconfig}"
kubectl rollout status daemonset/antrea-agent -n kube-system "${kubeconfig}"
}

function wait_for_multicluster_controller_ready {
kubectl create ns antrea-mcs-ns "${LEADER_CLUSTER_CONFIG}" || true
kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}"
kubectl rollout status deployment/antrea-mc-controller -n antrea-mcs-ns "${LEADER_CLUSTER_CONFIG}" || true
kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
kubectl create -f ./multicluster/test/yamls/leader-access-token-secret.yml "${LEADER_CLUSTER_CONFIG}" || true
kubectl get secret -n antrea-mcs-ns leader-access-token "${LEADER_CLUSTER_CONFIG}" -o yaml > ./multicluster/test/yamls/leader-access-token.yml

sed -i '/uid:/d' ./multicluster/test/yamls/leader-access-token.yml
sed -i '/resourceVersion/d' ./multicluster/test/yamls/leader-access-token.yml
sed -i '/last-applied-configuration/d' ./multicluster/test/yamls/leader-access-token.yml
sed -i '/type/d' ./multicluster/test/yamls/leader-access-token.yml
sed -i '/creationTimestamp/d' ./multicluster/test/yamls/leader-access-token.yml
sed -i 's/antrea-multicluster-member-access-sa/antrea-multicluster-controller/g' ./multicluster/test/yamls/leader-access-token.yml
sed -i 's/antrea-mcs-ns/kube-system/g' ./multicluster/test/yamls/leader-access-token.yml
echo "type: Opaque" >>./multicluster/test/yamls/leader-access-token.yml

kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member-only.yml "${EAST_CLUSTER_CONFIG}"
kubectl rollout status deployment/antrea-mc-controller -n kube-system "${EAST_CLUSTER_CONFIG}"
kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml "${EAST_CLUSTER_CONFIG}"
kubectl apply -f ./multicluster/test/yamls/east-member-cluster.yml "${EAST_CLUSTER_CONFIG}"

kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member-only.yml "${WEST_CLUSTER_CONFIG}"
kubectl rollout status deployment/antrea-mc-controller -n kube-system "${WEST_CLUSTER_CONFIG}"
kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml "${WEST_CLUSTER_CONFIG}"
kubectl apply -f ./multicluster/test/yamls/west-member-cluster.yml "${WEST_CLUSTER_CONFIG}"

kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}"
}

function wait_for_antrea_windows_pods_ready {
kubectl apply -f "${WORKDIR}/antrea.yml"
if [[ "${PROXY_ALL}" == false ]]; then
Expand Down Expand Up @@ -396,6 +490,78 @@ function deliver_antrea_windows {
rm -f antrea-windows.tar.gz
}

function deliver_antrea_multicluster {
echo "====== Building Antrea for the Following Commit ======"
export GO111MODULE=on
export GOPATH=${WORKDIR}/go
export GOROOT=/usr/local/go
export PATH=${GOROOT}/bin:$PATH

git show --numstat
make clean
${CLEAN_STALE_IMAGES}

cp -f build/yamls/*.yml $WORKDIR
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-ubuntu-all.sh --pull
echo "====== Delivering Antrea to all the Nodes ======"
docker save -o ${WORKDIR}/antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu:latest

kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG}| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${EAST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${WEST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true
done
}

function deliver_multicuster_controller {
echo "====== Build Antrea Multiple Cluster Controller and YAMLs ======"
export GO111MODULE=on
export GOPATH=${WORKDIR}/go
export GOROOT=/usr/local/go
export PATH=${GOROOT}/bin:$PATH

docker images | grep 'mc-controller' | awk '{print $3}' | xargs -r docker rmi || true
cd multicluster && mkdir -p bin && ln -s /usr/local/bin/kustomize ./bin/kustomize || true
make docker-build && cd ..

docker save antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar
./multicluster/hack/generate-manifest.sh -l antrea-mcs-ns >./multicluster/test/yamls/manifest.yml

kubectl get nodes -o wide --no-headers=true "${LEADER_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar
ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${EAST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${WEST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true
done

leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml
sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml

east_ip=$(kubectl get nodes -o wide --no-headers=true ${EAST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@[${east_ip}]:${WORKDIR}/serviceexport.yml

west_ip=$(kubectl get nodes -o wide --no-headers=true ${WEST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-west-serviceexport.yml jenkins@[${west_ip}]:${WORKDIR}/serviceexport.yml
}

function deliver_antrea {
echo "====== Cleanup Antrea Installation ======"
clean_up_one_ns "antrea-ipam-test"
Expand Down Expand Up @@ -613,6 +779,50 @@ function run_conformance_windows {
fi
}

function run_multicluster_e2e {
echo "====== Running Multicluster e2e Tests ======"
export GO111MODULE=on
export GOPATH=${WORKDIR}/go
export GOROOT=/usr/local/go
export GOCACHE=${WORKDIR}/.cache/go-build
export PATH=$GOROOT/bin:$PATH

wait_for_antrea_multicluster_pods_ready "${LEADER_CLUSTER_CONFIG}"
wait_for_antrea_multicluster_pods_ready "${EAST_CLUSTER_CONFIG}"
wait_for_antrea_multicluster_pods_ready "${WEST_CLUSTER_CONFIG}"

wait_for_multicluster_controller_ready

docker pull "${DOCKER_REGISTRY}/antrea/nginx"
docker tag "${DOCKER_REGISTRY}/antrea/nginx:latest" "nginx:latest"
docker save nginx:latest -o "${WORKDIR}"/nginx.tar

kubectl get nodes -o wide --no-headers=true "${LEADER_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/nginx.tar jenkins@["${IP}"]:"${WORKDIR}"/nginx.tar
ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/nginx.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${EAST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/nginx.tar jenkins@["${IP}"]:"${WORKDIR}"/nginx.tar
ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/nginx.tar" || true
done

kubectl get nodes -o wide --no-headers=true "${WEST_CLUSTER_CONFIG}"| awk '{print $6}' | while read IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/nginx.tar jenkins@["${IP}"]:"${WORKDIR}"/nginx.tar
ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/nginx.tar" || true
done

set +e
mkdir -p `pwd`/antrea-multicluster-test-logs
go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs
if [[ "$?" != "0" ]]; then
TEST_FAILURE=true
fi
set -e

tar -zcf antrea-test-logs.tar.gz antrea-test-logs
}

function run_install_windows_ovs {
echo "===== Verify Install-OVS ====="
export_govc_env_var
Expand Down Expand Up @@ -666,7 +876,12 @@ if [[ ${TESTCASE} == "windows-install-ovs" ]]; then
exit 0
fi

trap clean_antrea EXIT
if [[ ${TESTCASE} =~ "multicluster" ]]; then
trap clean_multicluster EXIT
else
trap clean_antrea EXIT
fi

if [[ ${TESTCASE} =~ "windows" ]]; then
deliver_antrea_windows
if [[ ${TESTCASE} =~ "e2e" ]]; then
Expand All @@ -677,6 +892,10 @@ if [[ ${TESTCASE} =~ "windows" ]]; then
elif [[ ${TESTCASE} =~ "e2e" ]]; then
deliver_antrea
run_e2e
elif [[ ${TESTCASE} =~ "multicluster" ]]; then
deliver_antrea_multicluster
deliver_multicuster_controller
run_multicluster_e2e
else
deliver_antrea
run_conformance
Expand Down
64 changes: 64 additions & 0 deletions multicluster/test/e2e/fixtures.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package e2e

import (
"os"
"path/filepath"
"testing"
)

func createDirectory(path string) error {
return os.Mkdir(path, 0700)
}

func (data *TestData) setupLogDirectoryForTest(testName string) error {
path := filepath.Join(testOptions.logsExportDir, testName)
// remove directory if it already exists. This ensures that we start with an empty
// directory
_ = os.RemoveAll(path)
err := createDirectory(path)
if err != nil {
return err
}
data.logsDirForTestCase = path
return nil
}

func setupTest(tb testing.TB) (*TestData, error) {
if err := testData.setupLogDirectoryForTest(tb.Name()); err != nil {
tb.Errorf("Error creating logs directory '%s': %v", testData.logsDirForTestCase, err)
return nil, err
}
success := false
defer func() {
if !success {
tb.Fail()
}
}()
tb.Logf("Creating '%s' K8s Namespace", multiClusterTestNamespace)
if err := testData.createTestNamespace(); err != nil {
return nil, err
}

success = true
return testData, nil
}

func teardownTest(tb testing.TB, data *TestData) {
if empty, _ := IsDirEmpty(data.logsDirForTestCase); empty {
_ = os.Remove(data.logsDirForTestCase)
}
}

func deletePodWrapper(tb testing.TB, data *TestData, clusterName string, namespace string, name string) {
tb.Logf("Deleting Pod '%s'", name)
if err := data.deletePod(clusterName, namespace, name); err != nil {
tb.Logf("Error when deleting Pod: %v", err)
}
}

func deleteServiceWrapper(tb testing.TB, data *TestData, clusterName string, namespace string, name string) {
tb.Logf("Deleting Service '%s'", name)
if err := data.deleteService(clusterName, namespace, name); err != nil {
tb.Logf("Error when deleting Service: %v", err)
}
}
Loading

0 comments on commit c1a7bfb

Please sign in to comment.