Merge pull request #766 from containerd/gh-pages-chart

.github/actions/run-hey-load-test/action.yml

@@ -20,27 +20,50 @@ runs:
       shell: bash
       run: |
         hey -n 100000 -c 50 http://127.0.0.1:8080 > raw-output.txt
-        python3 ./scripts/parse-hey.py raw-output.txt > benchmark-results.json
-        cat benchmark-results.json
+        python3 ./scripts/parse-hey.py raw-output.txt
+        cat latency_results.json
+        cat throughput_results.json
     - name: Report Throughput results
       uses: benchmark-action/[email protected]
       with:
         name: "HTTP Throughput"
         tool: "customBiggerIsBetter"
-        output-file-path: benchmark-results.json
+        output-file-path: throughput_results.json
         github-token: ${{ inputs.github-token }}
-        external-data-json-path: ./cache/benchmark-data.json
+        auto-push: ${{ github.event_name == 'schedule' }}
         summary-always: true
-        alert-threshold: "120%"
+        alert-threshold: "130%"
+        fail-on-alert: ${{ github.event_name == 'schedule' }}
+    - name: Report Latency results
+      if : ${{ github.event_name == 'schedule' }}
+      uses: benchmark-action/[email protected]
+      with:
+        name: "HTTP Latency"
+        tool: "customSmallerIsBetter"
+        output-file-path: latency_results.json
+        github-token: ${{ inputs.github-token }}
+        auto-push: true
+        summary-always: true
+        alert-threshold: "130%"
         fail-on-alert: true
+
+    # If the event is not a schedule, we'd run into a problem of the failed `git fetch` command,
+    # which attempts to update the local `gh-pages` branch with changes from the remote repository 
+    # but failed because the update is non-fast-forward. It failed because the previous step
+    # `benchmark-action/github-action-benchmark` has already committed the changes to the local, so
+    # it creates a conflict with the remote repository.
+    # 
+    # The workaround is to use the `external-data-json-path` option to tell the action to not 
+    # attempt to update the local `gh-pages` branch.
     - name: Report Latency results
+      if : ${{ github.event_name != 'schedule' }}
       uses: benchmark-action/[email protected]
       with:
         name: "HTTP Latency"
         tool: "customSmallerIsBetter"
-        output-file-path: benchmark-results.json
+        output-file-path: latency_results.json
         github-token: ${{ inputs.github-token }}
-        external-data-json-path: ./cache/benchmark-data.json
         summary-always: true
         alert-threshold: "130%"
-        fail-on-alert: true
+        fail-on-alert: false
+        external-data-json-path: ./cache/latency_results.json

.github/workflows/benchmarks.yml

@@ -5,6 +5,12 @@ on:
     - cron: '0 0 * * *'  # Runs daily at midnight
   pull_request:
 
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
 jobs:
   benchmark:
     runs-on: ubuntu-latest
@@ -24,18 +30,12 @@ jobs:
       - name: Build and load shims and wasi-demo-app
         shell: bash
         run: |
-          make build install test-image load test-image/oci load/oci test-image/http load/http
+          make build install test-image load test-image/oci load/oci
       - name: Run Benchmarks
         shell: bash
         run: |
           set -o pipefail
           cargo bench -p containerd-shim-benchmarks -- --output-format bencher | tee output.txt
-      # Download previous benchmark result from cache (if exists)
-      - name: Cache benchmark data
-        uses: actions/cache@v4
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
       - name: Store benchmark result
         uses: benchmark-action/[email protected]
         with:
@@ -47,12 +47,33 @@ jobs:
           # So I set the alert threshold to 130% of the previous benchmark result.
           # If the current benchmark result is more than 130% of the previous benchmark result, it will fail.
           alert-threshold: '130%'
-          fail-on-alert: true
+          fail-on-alert: ${{ github.event_name == 'schedule' }}
           alert-comment-cc-users: '@runwasi-committers'
           # Enable Job Summary
           summary-always: true
-          # Where the previous data file is stored
-          external-data-json-path: ./cache/benchmark-data.json
+          # Automatically push the benchmark result to gh-pages branch
+          # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details
+          auto-push: ${{ github.event_name == 'schedule' }}
+
+  benchmark-http:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch submodules
+        run: git submodule update --init --recursive
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: '' #Disable.  By default this action sets environment variable is set to -D warnings.  We manage this in the Makefile
+      - name: Setup build environment
+        shell: bash
+        run: |
+          os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
+          ./scripts/setup-$os.sh
+      - name: Build and load shims and wasi-demo-app
+        shell: bash
+        run: |
+          make build install test-image/http load/http
       - name: Start wasmtime shim
         shell: bash
         run: |
@@ -73,4 +94,4 @@ jobs:
         if: success()
         shell: bash
         run: |
-          sudo ctr task kill -s SIGKILL wasi-http
+          sudo ctr task kill -s SIGKILL wasi-http

scripts/parse-hey.py

@@ -57,19 +57,20 @@ def parse_hey_output(file_path):
 
     latency = latency * 1000 if latency is not None else None
 
-    results = []
     if rps is not None:
-        results.append({
+        throughput_result = [{
             "name": "HTTP RPS",
             "unit": "req/s",
             "value": rps
-        })
+        }]
+        with open('throughput_results.json', 'w') as f:
+            json.dump(throughput_result, f, indent=2)
 
     if latency is not None:
-        results.append({
+        latency_result = [{
             "name": "HTTP p95 Latency",
             "unit": "ms",
             "value": latency
-        })
-
-    print(json.dumps(results, indent=2))
+        }]
+        with open('latency_results.json', 'w') as f:
+            json.dump(latency_result, f, indent=2)