Merge pull request #1847 from navin772/calculate-coverage-codecov

feat(tests): Calculate coverage via codecov for k8s mode
kubearmor · Aug 24, 2024 · 550c032 · 550c032
2 parents bed86b5 + 753934b
commit 550c032
Show file tree

Hide file tree

Showing 7 changed files with 314 additions and 31 deletions.
diff --git a/.github/workflows/ci-merge-coverage.yaml b/.github/workflows/ci-merge-coverage.yaml
@@ -0,0 +1,67 @@
+name: ci-merge-coverage
+
+on:
+  workflow_run:
+    workflows: [ci-test-ginkgo]
+    types:
+      - completed
+
+jobs:
+  merge-coverage-files:
+    name: Download and merge files
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check if all required workflows completed successfully
+        id: check-workflows
+        run: |
+          workflows=("ci-test-ginkgo")
+          all_completed=true
+
+          commit_sha=$(git rev-parse HEAD)
+
+          for workflow in "${workflows[@]}"; do
+              conclusion=$(gh run list --workflow=$workflow --json conclusion,headSha,event,headBranch | jq -r --arg sha "$commit_sha" --arg event "pull_request" '.[] | select(.headSha == $sha and .event == $event) | .conclusion')
+
+              if [[ -z "$conclusion" ]]; then
+                  conclusion="pending"
+              fi
+
+              if [[ "$conclusion" != "success" ]]; then
+                  all_completed=false
+              fi
+
+              echo "${workflow}_status=$conclusion" >> $GITHUB_ENV
+          done
+
+          if [[ "$all_completed" == "true" ]]; then
+              echo "All workflows completed successfully for commit $commit_sha."
+              echo "all_succeeded=true" >> $GITHUB_ENV
+          else
+              echo "Not all workflows completed successfully for commit $commit_sha."
+              exit 1
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'KubeArmor/go.mod'
+
+      - name: Download k8s coverage files from ci-test-ginkgo
+        if: ${{ env.ci-test-ginkgo_status == 'success' }}
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          workflow: ci-test-ginkgo.yml
+          name: coverage.*
+          path: KubeArmor/
+          name_is_regexp: true
+          search_artifacts: true
+
+      - uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/ci-test-ginkgo.yml b/.github/workflows/ci-test-ginkgo.yml
@@ -62,6 +62,8 @@ jobs:
 
       - name: Generate KubeArmor artifacts
         run: |
+          #set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation
+          export IS_COVERAGE=true
           GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh
 
       - name: Build Kubearmor-Operator
@@ -82,8 +84,8 @@ jobs:
       - name: Run KubeArmor
         run: |
           if [[ ${{ matrix.runtime }} == "containerd" ]]; then
-            docker save kubearmor/kubearmor-init:latest | sudo k3s ctr images import -
-            docker save kubearmor/kubearmor:latest | sudo k3s ctr images import -
+            docker save kubearmor/kubearmor-test-init:latest | sudo k3s ctr images import -
+            docker save kubearmor/kubearmor-test:latest | sudo k3s ctr images import -
             docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import -
             docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import -
             
@@ -92,10 +94,10 @@ jobs:
             fi
           else
             if [ ${{ matrix.runtime }} == "crio" ]; then
-              docker save kubearmor/kubearmor-init:latest | sudo podman load
-              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-init:latest
-              docker save kubearmor/kubearmor:latest | sudo podman load
-              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor:latest
+              docker save kubearmor/kubearmor-test-init:latest | sudo podman load
+              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test-init:latest
+              docker save kubearmor/kubearmor-test:latest | sudo podman load
+              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test:latest
               docker save kubearmor/kubearmor-operator:latest | sudo podman load
               sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-operator:latest
               docker save kubearmor/kubearmor-snitch:latest | sudo podman load
@@ -112,25 +114,86 @@ jobs:
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A
           if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then
-            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml --dry-run=client -o json | \
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml --dry-run=client -o json | \
             jq '.spec.kubearmorControllerImage.imagePullPolicy = "Never"' | \
             kubectl apply -f -
           else 
-            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml
           fi
 
           kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test
           kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor
           kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor
           kubectl get pods -A
 
+          sleep 10
+          DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}')
+          echo "DaemonSet: $DAEMONSET_NAME"
+
+          kubectl patch daemonset $DAEMONSET_NAME -n kubearmor --type='json' -p='[
+            {
+              "op": "add",
+              "path": "/spec/template/spec/volumes/-",
+              "value": {
+                "name": "coverage-storage",
+                "hostPath": {
+                  "path": "/coverage",
+                  "type": "DirectoryOrCreate"
+                }
+              }
+            },
+            {
+              "op": "add",
+              "path": "/spec/template/spec/containers/0/volumeMounts/-",
+              "value": {
+                "mountPath": "/coverage",
+                "name": "coverage-storage"
+              }
+            },
+            {
+              "op": "add",
+              "path": "/spec/template/spec/containers/0/args/-",
+              "value": "-test.coverprofile=/coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out"
+            }
+          ]'
+
+          sleep 15
+
+      - name: Get KubeArmor POD info
+        run: |
+          DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}')
+          LABEL_SELECTOR=$(kubectl get daemonset $DAEMONSET_NAME -n kubearmor -o jsonpath='{.spec.selector.matchLabels}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' | paste -sd, -)
+          POD_NAME=$(kubectl get pods -n kubearmor -l "$LABEL_SELECTOR" -o jsonpath='{.items[*].metadata.name}')
+          echo "Pod: $POD_NAME"
+          echo "POD_NAME=$POD_NAME" >> $GITHUB_ENV
+
       - name: Test KubeArmor using Ginkgo
         run: |
           go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
           make
         working-directory: ./tests/k8s_env
         timeout-minutes: 30
 
+      - name: Kill KubeArmor prcoess in the pod
+        run: |
+          KUBEARMOR_PID=$(kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "ps aux | grep '[K]ubeArmor/kubearmor-test' | awk '{print \$1}'")
+          kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "kill -s SIGINT $KUBEARMOR_PID"
+          sleep 10
+        env:
+          POD_NAME: ${{ env.POD_NAME }}
+
+      - name: Extract coverage file
+        run: |
+          for i in {1..24}; do
+            if [ -f /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out ]; then
+              cp /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out
+              break
+            fi
+            sleep 5
+          done
+          ls -l
+        working-directory: KubeArmor
+
       - name: Get karmor sysdump
         if: ${{ failure() }}
         run: |
@@ -150,14 +213,15 @@ jobs:
       - name: Measure code coverage
         if: ${{ always() }}
         run: |
-          go install github.com/modocache/gover@latest
-          gover
-          go tool cover -func=gover.coverprofile
+          ls -l
+          go tool cover -func coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out
         working-directory: KubeArmor
         env:
           GOPATH: /home/runner/go
 
-      - uses: codecov/codecov-action@v3
+      - name: Upload coverage file
         if: ${{ always() }}
+        uses: actions/upload-artifact@v4
         with:
-          files: ./KubeArmor/gover.coverprofile
+          name: coverage-k8s-${{ matrix.os }}-${{ matrix.runtime }}
+          path: KubeArmor/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@
 
 ### Builder
 
-FROM golang:1.22-alpine3.20 as builder
+FROM golang:1.22-alpine3.20 AS builder
 
 RUN apk --no-cache update
 RUN apk add --no-cache git clang llvm make gcc protobuf
@@ -38,9 +38,15 @@ COPY ./KubeArmor/BPF .
 
 RUN make
 
+### Builder test
+
+FROM builder AS builder-test
+WORKDIR /usr/src/KubeArmor/KubeArmor
+RUN CGO_ENABLED=0 go test -covermode=atomic -coverpkg=./... -c . -o kubearmor-test
+
 ### Make executable image
 
-FROM alpine:3.20 as kubearmor
+FROM alpine:3.20 AS kubearmor
 
 RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories
 
@@ -53,6 +59,11 @@ COPY --from=builder /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/template
 
 ENTRYPOINT ["/KubeArmor/kubearmor"]
 
+FROM kubearmor AS kubearmor-test
+COPY --from=builder-test /usr/src/KubeArmor/KubeArmor/kubearmor-test /KubeArmor/kubearmor-test
+
+ENTRYPOINT ["/KubeArmor/kubearmor-test"]
+
 ### TODO ###
 
 ### build apparmor_parser binary
@@ -65,7 +76,7 @@ ENTRYPOINT ["/KubeArmor/kubearmor"]
 
 ### Make UBI-based executable image
 
-FROM redhat/ubi9-minimal as kubearmor-ubi
+FROM redhat/ubi9-minimal AS kubearmor-ubi
 
 ARG VERSION=latest
 ENV KUBEARMOR_UBI=true
@@ -100,4 +111,35 @@ RUN setcap "cap_sys_admin=ep cap_sys_ptrace=ep cap_ipc_lock=ep cap_sys_resource=
 USER 1000
 ENTRYPOINT ["/KubeArmor/kubearmor"]
 
+### Make UBI-based test executable image for coverage calculation
+FROM redhat/ubi9-minimal AS kubearmor-ubi-test
+
+ARG VERSION=latest
+ENV KUBEARMOR_UBI=true
 
+LABEL name="kubearmor" \
+      vendor="Accuknox" \
+      version=${VERSION} \
+      release=${VERSION} \
+      summary="kubearmor container image based on redhat ubi" \
+      description="KubeArmor is a cloud-native runtime security enforcement system that restricts the behavior \
+                  (such as process execution, file access, and networking operations) of pods, containers, and nodes (VMs) \
+                  at the system level."
+
+RUN microdnf -y update && \
+    microdnf -y install --nodocs --setopt=install_weak_deps=0 --setopt=keepcache=0 shadow-utils procps libcap && \
+    microdnf clean all
+
+RUN groupadd --gid 1000 default \
+  && useradd --uid 1000 --gid default --shell /bin/bash --create-home default
+
+COPY LICENSE /licenses/license.txt
+COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor
+COPY --from=builder --chown=default:default /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/
+COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/
+COPY --from=builder-test --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor-test /KubeArmor/kubearmor-test
+
+RUN setcap "cap_sys_admin=ep cap_sys_ptrace=ep cap_ipc_lock=ep cap_sys_resource=ep cap_dac_override=ep cap_dac_read_search=ep" /KubeArmor/kubearmor-test
+
+USER 1000
+ENTRYPOINT ["/KubeArmor/kubearmor-test"]
diff --git a/KubeArmor/build/build_kubearmor.sh b/KubeArmor/build/build_kubearmor.sh
@@ -44,6 +44,46 @@ echo "[INFO] Removed existing $REPO images"
 unset LABEL
 [[ "$GITHUB_SHA" != "" ]] && LABEL="--label github_sha=$GITHUB_SHA"
 
+# set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation
+if [[ "$IS_COVERAGE" == "true" ]]; then
+    REPO="kubearmor/kubearmor-test"
+
+    # build a kubearmor-test image
+    DTAG="-t $REPO:$VERSION"
+    echo "[INFO] Building $DTAG"
+    cd $ARMOR_HOME/..; docker build $DTAG -f Dockerfile --target kubearmor-test . $LABEL
+
+    if [ $? != 0 ]; then
+        echo "[FAILED] Failed to build $REPO:$VERSION"
+        exit 1
+    fi
+    echo "[PASSED] Built $REPO:$VERSION"
+
+    # build a kubearmor-test-init image
+    DTAGINI="-t $REPO-init:$VERSION"
+    echo "[INFO] Building $DTAGINI"
+    cd $ARMOR_HOME/..; docker build $DTAGINI -f Dockerfile.init --build-arg VERSION=$VERSION --target kubearmor-init . $LABEL
+
+    if [ $? != 0 ]; then
+        echo "[FAILED] Failed to build $REPO-init:$VERSION"
+        exit 1
+    fi
+    echo "[PASSED] Built $REPO-init:$VERSION"
+
+    # build kubearmor-ubi-test image
+    DTAGUBITEST="-t $UBIREPO-test:$VERSION"
+    echo "[INFO] Building $DTAGUBITEST"
+    cd $ARMOR_HOME/..; docker build $DTAGUBITEST -f Dockerfile --target kubearmor-ubi-test . $LABEL
+
+    if [ $? != 0 ]; then
+        echo "[FAILED] Failed to build $DTAGUBITEST:$VERSION"
+        exit 1
+    fi
+    echo "[PASSED] Built $DTAGUBITEST:$VERSION"
+
+    exit 0
+fi
+
 # build a kubearmor image
 DTAG="-t $REPO:$VERSION"
 echo "[INFO] Building $DTAG"