openyurtio · rambohe-ch · Sep 30, 2021 · Sep 30, 2021
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -112,3 +112,27 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - name: Build All Binaries
         run: make build
+  e2e-tests:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+
+      - name: Install Required Commands
+        run: |
+          go get sigs.k8s.io/[email protected]
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+      - name: Local Up Openyurt Cluster With Kind
+        run: bash hack/local_up_openyurt.sh
+
+      - name: Run e2e Tests
+        run: make e2e-tests
diff --git a/Makefile b/Makefile
@@ -62,3 +62,6 @@ clean:
 
 e2e:
 	hack/make-rules/build-e2e.sh
+
+e2e-tests:
+	bash hack/run-e2e-tests.sh
diff --git a/hack/kind-config-template.yaml b/hack/kind-config-template.yaml
@@ -0,0 +1,11 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+name: openyurt-e2e-test
+nodes:
+  - role: control-plane
+    image: |fill image here|
+    extraMounts:
+    - hostPath: |fill local bin dir|
+      containerPath: /root/openyurt
+  - role: worker
+    image: |fill image here|
diff --git a/hack/lib/release-images.sh b/hack/lib/release-images.sh
@@ -39,11 +39,25 @@ readonly -a SUPPORTED_ARCH=(
 
 readonly SUPPORTED_OS=linux
 
-readonly -a bin_targets=(${WHAT:-${YURT_BIN_TARGETS[@]}})
+readonly -a bin_targets=(${WHAT[@]:-${YURT_BIN_TARGETS[@]}})
 readonly -a bin_targets_process_servant=("${bin_targets[@]/yurtctl-servant/yurtctl}")
-readonly -a target_arch=(${ARCH:-${SUPPORTED_ARCH[@]}})
+readonly -a target_arch=(${ARCH[@]:-${SUPPORTED_ARCH[@]}})
 readonly region=${REGION:-us}
 
+# Parameters
+# $1: component name
+# $2: arch
+function get_image_name {
+    # If ${GIT_COMMIT} is not at a tag, add commit to the image tag. 
+    if [[ -z $(git tag --points-at ${GIT_COMMIT}) ]]; then
+        yurt_component_image="${REPO}/$1:${TAG}-$2-$(expr substr ${GIT_COMMIT} 1 7)"
+    else
+        yurt_component_image="${REPO}/$1:${TAG}-$2"
+    fi    
+
+    echo ${yurt_component_image}
+}
+
 function build_multi_arch_binaries() {
     local docker_run_opts=(
         "-i"
@@ -123,7 +137,7 @@ ENTRYPOINT ["/usr/local/bin/${binary_name}"]
 EOF
                fi
 
-               yurt_component_image="${REPO}/${yurt_component_name}:${TAG}-${arch}"
+               yurt_component_image=$(get_image_name ${yurt_component_name} ${arch})
                ln "${binary_path}" "${docker_build_path}/${binary_name}"
                docker build --no-cache -t "${yurt_component_image}" -f "${docker_file_path}" ${docker_build_path}
                docker save ${yurt_component_image} > ${YURT_IMAGE_DIR}/${yurt_component_name}-${SUPPORTED_OS}-${arch}.tar

diff --git a/hack/local_up_openyurt.sh b/hack/local_up_openyurt.sh
@@ -0,0 +1,286 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The OpenYurt Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This shell will create a openyurt cluster locally with kind. The yurt-tunnel will be
+# automatically deployed, and the autonomous mode will be active.
+#
+# It uses the following env variables:
+# REGION
+# Affect the GOPROXY. You can set it to "cn" to use GOPROXY="https://goproxy.cn".
+# Default value is "us", which means using GOPROXY="https://goproxy.io"
+# The default value is "us".
+#
+# KIND_KUBECONFIG
+# KIND_KUBECONFIG represents the path to store the kubeconfig file of the cluster
+# which is created by this shell. The default value is "$HOME/.kube/config"
+#
+# NODES_NUM
+# NODES_NUM represents the number of nodes to be included in the new-created cluster.
+# There are one control-plane node and NODES_NUM-1 worker nodes. Thus, NODES_NUM must
+# be greater than 2. The default value is 2.
+#
+# KUBERNETESVERSION
+# It declares the kubernetes version the cluster will use. The format is "1.XX". 
+# Now only 1.18, 1.19 and 1.20 are supported. The default value is 1.20
+#
+# TIMEOUT
+# TIMEOUT represents the time to wait for the kind control-plane, yurt-tunnel and
+# yurt-server to be ready. If it is not ready after the duration, the shell will exit.
+# The default value is 120s.
+#
+# YURTTUNNEL
+# If set YURTTUNNEL=disable, the yurt-tunnel-agent and yurt-tunnel-server will not be
+# deployed in the openyurt cluster.
+# The default value is "enable".
+
+
+set -x
+set -e
+set -u
+
+YURT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+KIND_KUBECONFIG=${KIND_KUBECONFIG:-${HOME}/.kube/config}
+
+readonly KIND_NODE_IMAGES=(
+    kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c
+    kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
+    kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
+)
+
+readonly REQUIRED_CMD=(
+    go
+    docker
+    kubectl
+    kind
+)
+
+readonly BUILD_TARGETS=(
+    yurthub
+    yurt-controller-manager
+    yurtctl
+    yurt-tunnel-server
+    yurt-tunnel-agent
+)
+
+readonly LOCAL_ARCH=$(go env GOHOSTARCH)
+readonly LOCAL_OS=$(go env GOHOSTOS)
+readonly IMAGES_DIR=${YURT_ROOT}/_output/images
+readonly CLUSTER_NAME="openyurt-e2e-test"
+readonly TIMEOUT=${TIMEOUT:-"120s"}
+readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"1.20"}
+readonly KIND_CONFIG=${YURT_ROOT}/_output/kind-config-v${KUBERNETESVERSION}.yaml
+readonly NODES_NUM=${NODES_NUM:-2}
+readonly KIND_CONTEXT="kind-${CLUSTER_NAME}"
+
+master=
+edgenodes=
+
+# $1 string to escape
+function escape_slash {
+    echo $1 | sed "s/\//\\\\\//g"
+}
+
+function gen_kind_config {
+    # check if number of nodes is valid
+    if [[ ${NODES_NUM} -lt 2 ]]; then
+        echo "NODES_NUM should be greater than 2"
+        exit -1 
+    fi
+
+    # check if kubernetes version is valid
+    local k8s_version=($(echo ${KUBERNETESVERSION} | sed "s/\./ /g"))
+    local major=${k8s_version[0]}
+    local minor=${k8s_version[1]}
+    if [[ ${major} -ne 1 ]] || [[ ${minor} -gt 20 ]] || [[ ${minor} -lt 18 ]]; then
+        echo "Invalid KUBERNETESVERSION, it should be between 1.18 and 1.20."
+        exit -1
+    fi
+
+    # create and init kind config file
+    local gen_config_path=${KIND_CONFIG}
+    cat ${YURT_ROOT}/hack/kind-config-template.yaml > ${gen_config_path}
+
+    # add additional node spec into kind config
+    for ((count=2; count<${NODES_NUM}; count++)); do
+        echo -e "\n  - role: worker\n    image: |fill image here|" >> ${gen_config_path}
+    done
+
+    # fill name:tag of images and fill bin dir
+    local bindir=$(escape_slash ${YURT_LOCAL_BIN_DIR}/${LOCAL_OS}/${LOCAL_ARCH})
+    local node_image=$(escape_slash $(echo ${KIND_NODE_IMAGES[${minor}-18]}))
+    sed -i "s/image: |fill image here|$/image: ${node_image}/g 
+        s/- hostPath: |fill local bin dir|/- hostPath: ${bindir}/g" \
+            ${gen_config_path}    
+}
+
+function install_kind {
+    echo "Begin to install kind"
+    GO111MODULE="on" go get sigs.k8s.io/[email protected]
+}
+
+function install_docker {
+    echo "docker should be installed first"
+    return -1
+}
+
+function install_kubectl {
+    echo "kubectl should be installed first"
+    return -1
+} 
+
+function install_go {
+    echo "go should be installed first"
+    return -1
+}
+
+function preflight {
+    echo "Preflight Check..."
+    for bin in "${REQUIRED_CMD[@]}"; do
+        command -v ${bin} > /dev/null 2>&1
+        if [[ $? -ne 0 ]]; then
+            echo "Cannot find command ${bin}."
+            install_${bin}
+            if [[ $? -ne 0 ]]; then
+                echo "Error occurred, exit"
+                exit -1
+            fi
+        fi
+    done
+}
+
+function build_target_binaries_and_images {
+    echo "Begin to build binaries and images"
+
+    export WHAT=${BUILD_TARGETS[@]}
+    export ARCH=${LOCAL_ARCH}
+
+    source ${YURT_ROOT}/hack/make-rules/release-images.sh    
+}
+
+function kind_load_images {
+    local postfix="${LOCAL_OS}-${LOCAL_ARCH}.tar"
+
+    for bin in ${BUILD_TARGETS[@]}; do
+        local imagename="${bin}-${postfix}"
+
+        if [[ "${bin}" = "yurtctl" ]]; then
+            imagename="yurtctl-servant-${postfix}"
+        fi
+
+        echo "loading image ${imagename} to nodes"
+        local nodesarg=$(echo "${master} ${edgenodes[@]}" | sed "s/ /,/g")
+        kind load image-archive ${IMAGES_DIR}/${imagename} \
+            --name ${CLUSTER_NAME} --nodes ${nodesarg}
+    done
+}
+
+function local_up_cluster {
+    echo "Creating kubernetes cluster with ${NODES_NUM} nodes"
+    gen_kind_config
+
+    # output kind config before kind cmd for the convenience of debugging
+    echo $(cat ${KIND_CONFIG})
+    kind create cluster --config ${KIND_CONFIG}
+
+    echo "Waiting for the control-plane ready..."
+    kubectl wait --for=condition=Ready node/${CLUSTER_NAME}-control-plane --context ${KIND_CONTEXT} --timeout=${TIMEOUT}
+    master=$(kubectl get node -A -o custom-columns=NAME:.metadata.name --context ${KIND_CONTEXT} | grep control-plane)
+    edgenodes=$(kubectl get node -A -o custom-columns=NAME:.metadata.name --context ${KIND_CONTEXT} | grep work)
+}
+
+# $1 yurt-tunnel-agent image with the format of "name:tag"
+function deploy_yurt_tunnel_agent {
+    local yurt_tunnel_agent_yaml=${YURT_OUTPUT_DIR}/yurt-tunnel-agent.yaml
+    # revise the yurt-tunnel-agent.yaml
+    cat ${YURT_ROOT}/config/setup/yurt-tunnel-agent.yaml |
+        sed "s/image: openyurt\/yurt-tunnel-agent:latest/image: $(escape_slash ${1})/" |
+            tee ${yurt_tunnel_agent_yaml}
+
+    kubectl apply -f ${yurt_tunnel_agent_yaml} --context ${KIND_CONTEXT}
+}
+
+# $1 yurt-tunnel-server image with the format of "name:tag"
+function deploy_yurt_tunnel_server {
+    local yurt_tunnel_server_yaml=${YURT_OUTPUT_DIR}/yurt-tunnel-server.yaml
+    # revise the yurt-tunnel-server.yaml
+    cat ${YURT_ROOT}/config/setup/yurt-tunnel-server.yaml | 
+        sed "s/image: openyurt\/yurt-tunnel-server:latest/image: $(escape_slash ${1})/;
+            s/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/" | 
+                tee ${yurt_tunnel_server_yaml}
+
+    kubectl apply -f ${yurt_tunnel_server_yaml} --context ${KIND_CONTEXT}
+}
+
+function deploy_yurt_tunnel {
+    deploy_yurt_tunnel_agent $(get_image_name "yurt-tunnel-agent" ${LOCAL_ARCH})
+    deploy_yurt_tunnel_server $(get_image_name "yurt-tunnel-server" ${LOCAL_ARCH})
+}
+
+function convert_to_openyurt {
+    echo "converting to openyurt cluster "
+
+    local yurtctl_dir="/root/openyurt"
+    # local yurtctl_dir=${YURT_LOCAL_BIN_DIR}/${LOCAL_OS}/${LOCAL_ARCH}
+    docker exec ${master} \
+        ${yurtctl_dir}/yurtctl convert --provider kubeadm --cloud-nodes ${master} \
+            --yurthub-image=$(get_image_name "yurthub" ${LOCAL_ARCH}) \
+            --yurt-controller-manager-image=$(get_image_name "yurt-controller-manager" ${LOCAL_ARCH}) \
+            --yurtctl-servant-image=$(get_image_name "yurtctl-servant" ${LOCAL_ARCH})
+            # --yurt-tunnel-server-image=$(get_image_name "yurt-tunnel-server" ${LOCAL_ARCH}) \
+            # --yurt-tunnel-agent-image=$(get_image_name "yurt-tunnel-agent" ${LOCAL_ARCH})
+
+    local nodearg=$(echo ${edgenodes[@]} | sed "s/ /,/g")
+    docker exec ${master} \
+        ${yurtctl_dir}/yurtctl markautonomous -a "${nodearg}"
+
+    if [ ! "${YURTTUNNEL:-"enable"}" = "disable" ]; then
+        deploy_yurt_tunnel    
+        kubectl rollout status deploy yurt-tunnel-server -n kube-system \
+            --timeout=${TIMEOUT} --context ${KIND_CONTEXT}
+        kubectl rollout status daemonset yurt-tunnel-agent -n kube-system \
+            --timeout=${TIMEOUT} --context ${KIND_CONTEXT}
+    fi
+}
+
+function get_kubeconfig {
+    mkdir -p ${HOME}/.kube 
+    kind get kubeconfig --name ${CLUSTER_NAME} > ${KIND_KUBECONFIG} 
+}
+
+function cleanup {
+    rm -rf ${YURT_ROOT}/_output
+    rm -rf ${YURT_ROOT}/dockerbuild
+    rm -f ${KIND_CONFIG} 
+    kind delete clusters ${CLUSTER_NAME}
+}
+
+function cleanup_on_err {
+    if [[ $? -ne 0 ]]; then
+        cleanup
+    fi
+}
+
+
+trap cleanup_on_err EXIT
+
+cleanup
+preflight
+build_target_binaries_and_images
+local_up_cluster
+kind_load_images
+convert_to_openyurt
+get_kubeconfig