Support Jenkins CI

This patch adds a Jenkins file example, it also modifies the E2E kind script to support a new mode of operation: the bash service. The modification was done to support running the scripts in a limited privilege shell.
abdallahyas · Jul 15, 2021 · a62d869 · a62d869
1 parent 7ce6ce2
commit a62d869
Show file tree

Hide file tree

Showing 7 changed files with 518 additions and 8 deletions.
diff --git a/ci/admin-list.yaml b/ci/admin-list.yaml
@@ -0,0 +1,5 @@
+admin-list:
+  - mellanox-ci
+org-list:
+  - Mellanox
+
diff --git a/ci/jenkins/sriov-network-operator-ci.yaml b/ci/jenkins/sriov-network-operator-ci.yaml
@@ -0,0 +1,91 @@
+- project:
+    name: sriov-network-operator-github-ci
+    jobs:
+      - 'sriov-network-operator-ci':
+          project: sriov-network-operator
+          disabled_var: false
+          concurrent: false
+          node: GITHUB
+          git-site: https://github.com
+          git-root: AbdYsn
+          git-project: sriov-network-operator
+
+- job-template:
+    name: 'sriov-network-operator-ci'
+    node: '{node}'
+    builders:
+      - inject:
+          properties-content: |
+            KUBECONFIG=/etc/kubernetes/admin.conf
+            INTERFACES_SWITCHER=bash_service
+      - run-e2e-test
+    concurrent: false
+    description: <!-- Managed by Jenkins Job Builder -->
+    disabled: false
+    project-type: freestyle
+    properties:
+      - build-discarder:
+          artifact-days-to-keep: 60
+          artifact-num-to-keep: 100
+          days-to-keep: 60
+          num-to-keep: 100
+      - github:
+          url: '{git-site}/{git-root}/{git-project}'
+    scm:
+      - git:
+          branches: ["${{sha1}}"]
+          credentials-id: '{credentials-id}'
+          name: '{git-project}'
+          refspec: +refs/pull/*:refs/remotes/origin/pr/*
+          url: '{git-site}/{git-root}/{git-project}'
+          wipe-workspace: true
+    triggers:
+      - github-pull-request:
+          admin-list:
+            - mellanox-ci
+          allow-whitelist-orgs-as-admins: true
+          org-list:
+            - Mellanox
+          auth-id: '{auth-id}'
+          auto-close-on-fail: false
+          build-desc-template: null
+          cron: H/5 * * * *
+          github-hooks: false
+          only-trigger-phrase: true
+          cancel-builds-on-update: true
+          permit-all: false
+          status-url: --none--
+          success-status: "Build Passed"
+          failure-status: "Build Failed, comment `/test-e2e` or `/test-all` to retrigger"
+          error-status: "Build Failed, comment `/test-e2e` or `/test-all` to retrigger"
+          status-context: '{project} CI'
+          trigger-phrase: ".*/test-(all|e2e(,| |$)).*"
+          white-list:
+            - '*'
+          white-list-target-branches:
+            - master
+      - github
+    wrappers:
+      - timeout:
+          timeout: 120
+          fail: true
+      - timestamps
+
+- builder:
+    name: run-e2e-test
+    builders:
+      - shell: |
+          #!/bin/bash
+          status=0
+          ./hack/teardown-e2e-kind-cluster.sh
+          sleep 5
+
+          mlnx_pci=$(lspci | grep Mellanox | grep -Ev 'MT27500|MT27520|Virt' | head -n 1 | awk '{print $1}')
+          ./hack/run-e2e-test-kind.sh 0000:${mlnx_pci}
+          let status=$status+$?
+
+          ./hack/teardown-e2e-kind-cluster.sh
+          sleep 5
+
+          exit $status
+
diff --git a/hack/e2e.md b/hack/e2e.md
@@ -0,0 +1,46 @@
+# E2E Tests
+E2E tests are used to deploy an sriovnetworknodepolicy on an existing Kubernetes cluster and make sure it creates VFs using different configurations.
+To run the tests on a Kubernetes cluster run:
+```
+  ./hack/run-e2e-test.sh
+```
+
+# E2E tests using KIND
+Kubernetes IN Docker (KIND) is a tool to deploy Kubernetes inside Docker containers. It is used to test multi nodes scenarios on a single baremetal node. 
+To run the E2E tests inside a KIND cluster, `./hack/run-e2e-test-kind.sh` can be used. The script deploys a KIND cluster, switch the specified interface to the kind-worker namespace, deploys the operator, and run the E2E tests. There are two modes of operation for the E2E KIND scripts depending on the mechanism used to switch the specified PF into the KIND cluster:
+ * `go` mode (default): In this mode, the E2E test suit handle the PF and its VFs switching to the test namespace.
+ * `bash_service` mode: In this mode a dedicated bash service is used to switch the PF and VFs to the test namespace.
+
+`kind` tool need to be installed on the server, follow [kind documentation](https://kind.sigs.k8s.io/docs/user/quick-start/) to install kind.
+
+## Running E2E kind tests in `go` mode
+To Run the E2E tests in a kind cluster in go mode. In a root shell simply run the E2E kind script:
+```
+./hack/run-e2e-test-kind.sh <interface pci>
+```
+## Running the E2E kind tests in `bash_service` mode
+The `bash_service` mode uses a bash service to handle the interface switching. To prepare the service, the following needs to be done as root:
+```
+cp ./hack/vf-netns-switcher.sh /usr/bin/
+cp ./hack/vf-switcher.service /etc/systemd/system/
+systemctl daemon-reload
+```
+For the service to work probably the `yq` tool is needed. To install the tool use:
+```
+wget https://github.com/mikefarah/yq/releases/download/3.4.0/yq_linux_amd64 -O /usr/bin/yq
+chmod +x /usr/bin/yq
+```
+
+To run the E2E tests do:
+```
+KUBECONFIG=/etc/kubernetes/admin.conf
+INTERFACES_SWITCHER=bash_service
+./hack/run-e2e-test-kind.sh <interface pci>
+```
+
+## Teardown the KIND cluster
+To cleanup the KIND cluster use:
+```
+./hack/teardown-e2e-kind-cluster.sh
+```
+
diff --git a/hack/run-e2e-test-kind.sh b/hack/run-e2e-test-kind.sh
@@ -4,6 +4,8 @@ here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")"
 root="$(readlink --canonicalize "$here/..")"
 export SRIOV_NETWORK_OPERATOR_IMAGE="${SRIOV_NETWORK_OPERATOR_IMAGE:-sriov-network-operator:latest}"
 export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE:-origin-sriov-network-config-daemon:latest}"
+export KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config}"
+export INTERFACES_SWITCHER="${INTERFACES_SWITCHER:-"go"}"
 RETRY_MAX=10
 INTERVAL=10
 TIMEOUT=300
@@ -50,13 +52,14 @@ retry() {
 echo "## checking requirements"
 check_requirements
 echo "## delete any existing cluster, deploy control & data plane cluster with KinD"
-retry kind delete cluster &&  cat <<EOF | kind create cluster --config=-
+retry kind delete cluster &&  cat <<EOF | kind create cluster --kubeconfig=${KUBECONFIG} --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
   - role: control-plane
   - role: worker
 EOF
+sudo chmod 644 ${KUBECONFIG}
 echo "## build operator image"
 retry docker build -t "${SRIOV_NETWORK_OPERATOR_IMAGE}" -f "${root}/Dockerfile" "${root}"
 echo "## load operator image into KinD"
@@ -67,8 +70,6 @@ echo "## load daemon image into KinD"
 kind load docker-image "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}"
 echo "## export kube config for utilising locally"
 kind export kubeconfig
-echo "## exporting KUBECONFIG environment variable to access KinD K8 API server"
-export KUBECONFIG="${HOME}/.kube/config"
 echo "## wait for coredns"
 retry kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=${TIMEOUT}s
 echo "## install multus"
@@ -85,11 +86,21 @@ echo "## label KinD's control-plane-node as sriov capable"
 kubectl label node kind-worker feature.node.kubernetes.io/network-sriov.capable=true --overwrite
 echo "## label KinD worker as worker"
 kubectl label node kind-worker node-role.kubernetes.io/worker= --overwrite
-echo "## retrieving netns path from container"
-netns_path="$(docker inspect --format '{{ .NetworkSettings.SandboxKey }}' "${kind_container}")"
-echo "## exporting test device '${test_pf_pci_addr}' and test netns path '${netns_path}'"
-export TEST_PCI_DEVICE="${test_pf_pci_addr}"
-export TEST_NETNS_PATH="${netns_path}"
+if [[ "${INTERFACES_SWITCHER}" == "bash_service" ]];then
+    pf="$(ls /sys/bus/pci/devices/${test_pf_pci_addr}/net)"
+    cat <<EOF > /etc/vf-switcher/vf-switcher.yaml
+- netns: ${kind_container}
+  pfs:
+    - ${pf}
+EOF
+    sudo systemctl restart vf-switcher.service
+else
+    echo "## retrieving netns path from container"
+    netns_path="$(sudo docker inspect --format '{{ .NetworkSettings.SandboxKey }}' "${kind_container}")"
+    echo "## exporting test device '${test_pf_pci_addr}' and test netns path '${netns_path}'"
+    export TEST_PCI_DEVICE="${test_pf_pci_addr}"
+    export TEST_NETNS_PATH="${netns_path}"
+fi
 echo "## disabling webhooks"
 export ENABLE_ADMISSION_CONTROLLER=false
 echo "## deploying SRIOV Network Operator"

diff --git a/hack/teardown-e2e-kind-cluster.sh b/hack/teardown-e2e-kind-cluster.sh
@@ -7,3 +7,5 @@ if ! command -v kind &> /dev/null; then
 fi
 
 kind delete cluster
+sudo systemctl stop vf-switcher.service
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,3 +7,5 @@ if ! command -v kind &> /dev/null; then
		fi

		kind delete cluster
		sudo systemctl stop vf-switcher.service