From f2a6009e7e710d8a2f9f85a31b91b961c5cbc642 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Wed, 16 Oct 2024 14:00:55 -0400 Subject: [PATCH 01/12] chore: check Signed-off-by: Case Wylie --- src/lib.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.ts b/src/lib.ts index 5d3a530b4..e7771de0a 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -8,7 +8,7 @@ import { PeprMutateRequest } from "./lib/mutate-request"; import * as PeprUtils from "./lib/utils"; import { PeprValidateRequest } from "./lib/validate-request"; import * as sdk from "./sdk/sdk"; - +console.log("HI"); export { Capability, K8s, From 44f37f97f8b3cff6fa362ad298baddf9e504d0bf Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:38:47 -0400 Subject: [PATCH 02/12] chore: soak interrupts test Signed-off-by: Case Wylie --- .github/workflows/soak-interrupts.yaml | 214 +++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 .github/workflows/soak-interrupts.yaml diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml new file mode 100644 index 000000000..bdcdacdbc --- /dev/null +++ b/.github/workflows/soak-interrupts.yaml @@ -0,0 +1,214 @@ +name: Soak Interrupts Test + +permissions: read-all +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to run the workflow on' + required: true + default: 'main' + http2: + description: 'use http2 watcher' + required: false + default: 'false' + schedule: + - cron: '0 4 * * *' # 12AM EST/9PM PST + +jobs: + pepr-build: + name: controller image + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + with: + egress-policy: audit + + - name: clone pepr + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + repository: defenseunicorns/pepr + path: pepr + + - name: "set env: PEPR" + run: echo "PEPR=${GITHUB_WORKSPACE}/pepr" >> "$GITHUB_ENV" + + - name: setup node + uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: pepr + + - name: install pepr deps + run: | + cd "$PEPR" + npm ci + + - name: build pepr image + run: | + cd "$PEPR" + npm run build:image + + - name: tar pepr image + run: | + PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar" + echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV" + docker image save --output "$PEPR_TAR" pepr:dev + + - name: upload image tar artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: pepr-img.tar + path: pepr-img.tar + retention-days: 1 + + soak-test: + name: soak-test + runs-on: ubuntu-latest + needs: + - pepr-build + + steps: + - name: Harden Runner + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + with: + egress-policy: audit + + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: "install k3d" + run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash" + shell: bash + + - name: dowload image tar artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: pepr-img.tar + path: ${{ github.workspace }} + + - name: import pepr image from tar + run: | + PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar" + echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV" + docker image load --input "$PEPR_TAR" + + - name: Create k3d Cluster + run: "k3d cluster create" + shell: bash + + - name: Import pepr image into k3d + run: "k3d image import pepr:dev -c k3s-default" + shell: bash + + - name: Install istioctl + run: | + curl -L https://istio.io/downloadIstio | sh - + shell: bash + + - name: Install default profile + run: | + cd istio*/bin + ./istioctl install --set profile=demo -y + + - name: Set up Kubernetes + uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 + with: + version: 'latest' + + - name: Create logs directory + run: mkdir -p logs + + - name: Deploy Pepr + if: ${{ (github.event.inputs.http2 || 'none') != 'true' }} + run: | + kubectl apply -f hack/soak.ci.yaml + + - name: Deploy http2 Pepr + if: ${{ (github.event.inputs.http2 || 'none') == 'true' }} + run: | + kubectl apply -f hack/soak-http2.ci.yaml + + - name: Deploy applications + run: | + kubectl apply -f hack/auditor.ci.yaml + + - name: Wait for 15 minutes before starting pod checks + run: | + sleep 10s + kubectl wait --for=condition=ready -n istio-system pod -l istio=pilot + kubectl wait --for=condition=ready -n istio-system pod -l app=istio-ingressgateway + kubectl wait --for=condition=ready -n watch-auditor pod -l app=watch-auditor + kubectl wait --for=condition=ready -n pepr-system pod -l app=pepr-soak-ci-watcher + + - name: Run the soak test and collect metrics + run: | + # Initialize the map to store pod counts + declare -A pod_map + + update_pod_map() { + for pod in $(kubectl get pods -n pepr-demo -o jsonpath='{.items[*].metadata.name}'); do + count=${pod_map[$pod]} + if [ -z "$count" ]; then + pod_map[$pod]=1 + else + pod_map[$pod]=$((count + 1)) + fi + done + } + touch logs/auditor-log.txt + touch logs/informer-log.txt + + update_pod_map + + collect_metrics() { + kubectl exec metrics-collector -n watch-auditor -- curl watch-auditor:8080/metrics | grep watch_controller_failures_total > logs/auditor-log.txt || true + kubectl exec metrics-collector -n watch-auditor -- curl -k https://pepr-soak-ci-watcher.pepr-system.svc.cluster.local/metrics | egrep -E "pepr_cache_miss|pepr_resync_failure_count" > logs/informer-log.txt + kubectl logs -n pepr-system deploy/pepr-soak-ci-watcher > logs/watch-log.txt + } + + # Start collecting metrics every 5 minutes and checking pod counts every 30 minutes + for i in {1..13}; do # 13 iterations cover 65 minutes (1 hours and 5 minutes) (Every 5 mins 5137 = 65 mins = 1 hour 5 mins) + collect_metrics + cat logs/informer-log.txt + cat logs/auditor-log.txt + if [ $((i % 2)) -eq 0 ]; then # Every 10 minutes + update_pod_map + + # get a list of pods every 10 mins + kubectl get pods -n pepr-demo + kubectl top po -n pepr-system + kubectl get po -n pepr-system + + # Verify that no pod's count exceeds 1 + for pod in "${!pod_map[@]}"; do + echo "$pod: ${pod_map[$pod]}" + if [ "${pod_map[$pod]}" -gt 1 ]; then + echo "Test failed: Pod $pod has count ${pod_map[$pod]}" + exit 1 + fi + done + + # Every 10 minutes, scale up or down the `watch-auditor` deployment + if [ $((i % 4)) -eq 0 ]; then + # On even iterations (e.g., i = 0, 4, 8...), scale down to 0 replicas + echo "Scaling down the watch-auditor deployment to 0 replicas" + kubectl scale deploy/watch-auditor -n watch-auditor --replicas=0 + else + # On odd iterations (e.g., i = 2, 6, 10...), scale up to 1 replica + echo "Scaling up the watch-auditor deployment to 0 replicas" + kubectl scale deploy/watch-auditor -n watch-auditor --replicas=1 + fi + fi + sleep 300s # Sleep for 5 minutes before the next iteration + done + + echo "Soak test passed successfully!" + shell: bash + + - name: Upload logs + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: soak-test-logs + path: logs From 659674d70a11395cc70553c99ef4533ab40f6b52 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:41:01 -0400 Subject: [PATCH 03/12] Update src/lib.ts --- src/lib.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib.ts b/src/lib.ts index e7771de0a..17f207f11 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -8,7 +8,6 @@ import { PeprMutateRequest } from "./lib/mutate-request"; import * as PeprUtils from "./lib/utils"; import { PeprValidateRequest } from "./lib/validate-request"; import * as sdk from "./sdk/sdk"; -console.log("HI"); export { Capability, K8s, From 8de0a48c26d599768b820d00c9a73357ad5ee0be Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:41:17 -0400 Subject: [PATCH 04/12] Update src/lib.ts --- src/lib.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.ts b/src/lib.ts index 17f207f11..5d3a530b4 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -8,6 +8,7 @@ import { PeprMutateRequest } from "./lib/mutate-request"; import * as PeprUtils from "./lib/utils"; import { PeprValidateRequest } from "./lib/validate-request"; import * as sdk from "./sdk/sdk"; + export { Capability, K8s, From 6773b65b8b4252bc86a221d198e9a3ff3df33c12 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:42:51 -0400 Subject: [PATCH 05/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index bdcdacdbc..bf2381191 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -169,7 +169,7 @@ jobs: } # Start collecting metrics every 5 minutes and checking pod counts every 30 minutes - for i in {1..13}; do # 13 iterations cover 65 minutes (1 hours and 5 minutes) (Every 5 mins 5137 = 65 mins = 1 hour 5 mins) + for i in {1..13}; do # 13 iterations cover 65 minutes (1 hours and 5 minutes) (Every 5 mins x 13 = 65 mins = 1 hour 5 mins) collect_metrics cat logs/informer-log.txt cat logs/auditor-log.txt From 4fd06aa5c1cb393d0511228f04816d8eb88f7532 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:46:45 -0400 Subject: [PATCH 06/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index bf2381191..8e87f467b 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -196,7 +196,6 @@ jobs: echo "Scaling down the watch-auditor deployment to 0 replicas" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=0 else - # On odd iterations (e.g., i = 2, 6, 10...), scale up to 1 replica echo "Scaling up the watch-auditor deployment to 0 replicas" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=1 fi From 13ff290e848de17cbd4ecfcdad55df343516e37d Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:46:58 -0400 Subject: [PATCH 07/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index 8e87f467b..49f26e82c 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -192,7 +192,6 @@ jobs: # Every 10 minutes, scale up or down the `watch-auditor` deployment if [ $((i % 4)) -eq 0 ]; then - # On even iterations (e.g., i = 0, 4, 8...), scale down to 0 replicas echo "Scaling down the watch-auditor deployment to 0 replicas" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=0 else From ecc419d6aa8533a3a3768b8e6f92c6ac1eb4d893 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 11:47:14 -0400 Subject: [PATCH 08/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index 49f26e82c..d4d43471e 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -190,7 +190,7 @@ jobs: fi done - # Every 10 minutes, scale up or down the `watch-auditor` deployment + # Every 20 minutes, scale up or down the `watch-auditor` deployment if [ $((i % 4)) -eq 0 ]; then echo "Scaling down the watch-auditor deployment to 0 replicas" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=0 From 5f5a696cd9c3e80e4bcdd7eb13518740bf6097fc Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 12:59:31 -0400 Subject: [PATCH 09/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index d4d43471e..d349932a4 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -64,7 +64,7 @@ jobs: path: pepr-img.tar retention-days: 1 - soak-test: + soak-interrupts-test: name: soak-test runs-on: ubuntu-latest needs: From 425857d7831463c6cb9e93d6fcb4cd28c879a955 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 12:59:35 -0400 Subject: [PATCH 10/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index d349932a4..3ff7916a8 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -65,7 +65,7 @@ jobs: retention-days: 1 soak-interrupts-test: - name: soak-test + name: soak-interrupts-test runs-on: ubuntu-latest needs: - pepr-build From 2b49f459298bed658a71a983af12658b3df7eb92 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 13:13:57 -0400 Subject: [PATCH 11/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index 3ff7916a8..43d19a3e8 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -195,7 +195,7 @@ jobs: echo "Scaling down the watch-auditor deployment to 0 replicas" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=0 else - echo "Scaling up the watch-auditor deployment to 0 replicas" + echo "Scaling up the watch-auditor deployment to 1 replica" kubectl scale deploy/watch-auditor -n watch-auditor --replicas=1 fi fi From 2635e73d61cc261f62c852ac5477c866d9662bb3 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Fri, 18 Oct 2024 13:51:32 -0400 Subject: [PATCH 12/12] Update .github/workflows/soak-interrupts.yaml --- .github/workflows/soak-interrupts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/soak-interrupts.yaml b/.github/workflows/soak-interrupts.yaml index 43d19a3e8..c1dc48cf5 100644 --- a/.github/workflows/soak-interrupts.yaml +++ b/.github/workflows/soak-interrupts.yaml @@ -202,7 +202,7 @@ jobs: sleep 300s # Sleep for 5 minutes before the next iteration done - echo "Soak test passed successfully!" + echo "Soak interrupt test passed successfully!" shell: bash - name: Upload logs