From 9e67edd96863409ac2c1a1a74e22546030a09915 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Fri, 6 Dec 2024 23:27:54 +0000 Subject: [PATCH 1/7] .github/workflows/benchmarks: support push to Github Pages this commit adds 'auto-push: true' to the benchmark-action/github-action-benchmark which allows it commit changes and push to gh-pages branch and display charts dashboard on GitHub Pages. Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/workflows/benchmarks.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1fe67489c..e6f10a2dd 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -5,6 +5,12 @@ on: - cron: '0 0 * * *' # Runs daily at midnight pull_request: +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + jobs: benchmark: runs-on: ubuntu-latest @@ -51,8 +57,9 @@ jobs: alert-comment-cc-users: '@runwasi-committers' # Enable Job Summary summary-always: true - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json + # Automatically push the benchmark result to gh-pages branch + # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details + auto-push: true - name: Start wasmtime shim shell: bash run: | @@ -73,4 +80,4 @@ jobs: if: success() shell: bash run: | - sudo ctr task kill -s SIGKILL wasi-http \ No newline at end of file + sudo ctr task kill -s SIGKILL wasi-http From 100607f953442ee65fc0b29c361bc1b517f3ac75 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Sat, 7 Dec 2024 00:02:18 +0000 Subject: [PATCH 2/7] .github/workflows/benchmarks: remove external-data-json-path since we added auto-push, we no longer need to fetch the cache and compare the result with the cache because each time the benchmark runs, it will generate a commit and save to the gh-pages branch. Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/workflows/benchmarks.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e6f10a2dd..90b61fac1 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -36,12 +36,6 @@ jobs: run: | set -o pipefail cargo bench -p containerd-shim-benchmarks -- --output-format bencher | tee output.txt - # Download previous benchmark result from cache (if exists) - - name: Cache benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1.20.4 with: From 547021bae4f4b01e3eddae49873967b813b06615 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 12 Dec 2024 05:51:03 +0000 Subject: [PATCH 3/7] .github: move HTTP benchmark to it's own job Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/actions/run-hey-load-test/action.yml | 4 ++-- .github/workflows/benchmarks.yml | 22 +++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index 5364cb04b..467c739d2 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -29,7 +29,7 @@ runs: tool: "customBiggerIsBetter" output-file-path: benchmark-results.json github-token: ${{ inputs.github-token }} - external-data-json-path: ./cache/benchmark-data.json + auto-push: true summary-always: true alert-threshold: "120%" fail-on-alert: true @@ -40,7 +40,7 @@ runs: tool: "customSmallerIsBetter" output-file-path: benchmark-results.json github-token: ${{ inputs.github-token }} - external-data-json-path: ./cache/benchmark-data.json + auto-push: true summary-always: true alert-threshold: "130%" fail-on-alert: true \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 90b61fac1..27010b2cb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -30,7 +30,7 @@ jobs: - name: Build and load shims and wasi-demo-app shell: bash run: | - make build install test-image load test-image/oci load/oci test-image/http load/http + make build install test-image load test-image/oci load/oci - name: Run Benchmarks shell: bash run: | @@ -54,6 +54,26 @@ jobs: # Automatically push the benchmark result to gh-pages branch # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details auto-push: true + + benchmark-http: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Fetch submodules + run: git submodule update --init --recursive + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile + - name: Setup build environment + shell: bash + run: | + os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') + ./scripts/setup-$os.sh + - name: Build and load shims and wasi-demo-app + shell: bash + run: | + make build install test-image/http load/http - name: Start wasmtime shim shell: bash run: | From 72d4ffa6709b6b096669888f9f0edb9e400a61aa Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Fri, 13 Dec 2024 00:11:34 +0000 Subject: [PATCH 4/7] .github: github-action-benchmark pushes to gh-pages branch only when event is 'schedule' this is so that PRs that run benchmarks do not push the results to the gh-pages branch to be shown on the /dev/bench website Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/actions/run-hey-load-test/action.yml | 8 ++++---- .github/workflows/benchmarks.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index 467c739d2..cc19af13c 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -29,10 +29,10 @@ runs: tool: "customBiggerIsBetter" output-file-path: benchmark-results.json github-token: ${{ inputs.github-token }} - auto-push: true + auto-push: {{ github.event_name == 'schedule' }} summary-always: true alert-threshold: "120%" - fail-on-alert: true + fail-on-alert: {{ github.event_name == 'schedule' }} - name: Report Latency results uses: benchmark-action/github-action-benchmark@v1.20.4 with: @@ -40,7 +40,7 @@ runs: tool: "customSmallerIsBetter" output-file-path: benchmark-results.json github-token: ${{ inputs.github-token }} - auto-push: true + auto-push: {{ github.event_name == 'schedule' }} summary-always: true alert-threshold: "130%" - fail-on-alert: true \ No newline at end of file + fail-on-alert: {{ github.event_name == 'schedule' }} \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 27010b2cb..a0af6ce26 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -47,13 +47,13 @@ jobs: # So I set the alert threshold to 130% of the previous benchmark result. # If the current benchmark result is more than 130% of the previous benchmark result, it will fail. alert-threshold: '130%' - fail-on-alert: true + fail-on-alert: {{ github.event_name == 'schedule' }} alert-comment-cc-users: '@runwasi-committers' # Enable Job Summary summary-always: true # Automatically push the benchmark result to gh-pages branch # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details - auto-push: true + auto-push: {{ github.event_name == 'schedule' }} benchmark-http: runs-on: ubuntu-latest From 43c575b8df47271b5a8ecc5f770043673ecd860f Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Fri, 13 Dec 2024 00:15:48 +0000 Subject: [PATCH 5/7] .github, scripts: Update parse-hey.py to output two json files instead of one The CI will generate two reports, one for the latency and another for the throughput corresponding to the two parsed .json files from the script Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/actions/run-hey-load-test/action.yml | 11 ++++++----- scripts/parse-hey.py | 15 ++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index cc19af13c..e02c0e7b0 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -20,25 +20,26 @@ runs: shell: bash run: | hey -n 100000 -c 50 http://127.0.0.1:8080 > raw-output.txt - python3 ./scripts/parse-hey.py raw-output.txt > benchmark-results.json - cat benchmark-results.json + python3 ./scripts/parse-hey.py raw-output.txt + cat latency_results.json + cat throughput_results.json - name: Report Throughput results uses: benchmark-action/github-action-benchmark@v1.20.4 with: name: "HTTP Throughput" tool: "customBiggerIsBetter" - output-file-path: benchmark-results.json + output-file-path: throughput_results.json github-token: ${{ inputs.github-token }} auto-push: {{ github.event_name == 'schedule' }} summary-always: true - alert-threshold: "120%" + alert-threshold: "130%" fail-on-alert: {{ github.event_name == 'schedule' }} - name: Report Latency results uses: benchmark-action/github-action-benchmark@v1.20.4 with: name: "HTTP Latency" tool: "customSmallerIsBetter" - output-file-path: benchmark-results.json + output-file-path: latency_results.json github-token: ${{ inputs.github-token }} auto-push: {{ github.event_name == 'schedule' }} summary-always: true diff --git a/scripts/parse-hey.py b/scripts/parse-hey.py index d946441da..438c166d6 100644 --- a/scripts/parse-hey.py +++ b/scripts/parse-hey.py @@ -57,19 +57,20 @@ def parse_hey_output(file_path): latency = latency * 1000 if latency is not None else None - results = [] if rps is not None: - results.append({ + throughput_result = [{ "name": "HTTP RPS", "unit": "req/s", "value": rps - }) + }] + with open('throughput_results.json', 'w') as f: + json.dump(throughput_result, f, indent=2) if latency is not None: - results.append({ + latency_result = [{ "name": "HTTP p95 Latency", "unit": "ms", "value": latency - }) - - print(json.dumps(results, indent=2)) + }] + with open('latency_results.json', 'w') as f: + json.dump(latency_result, f, indent=2) From b7f7e1b8ae530ee5d1106b179cc8a5ed5f076a41 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Fri, 13 Dec 2024 07:53:02 +0000 Subject: [PATCH 6/7] .github: fixed a type in terms of action syntax Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/actions/run-hey-load-test/action.yml | 8 ++++---- .github/workflows/benchmarks.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index e02c0e7b0..3050b1200 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -30,10 +30,10 @@ runs: tool: "customBiggerIsBetter" output-file-path: throughput_results.json github-token: ${{ inputs.github-token }} - auto-push: {{ github.event_name == 'schedule' }} + auto-push: ${{ github.event_name == 'schedule' }} summary-always: true alert-threshold: "130%" - fail-on-alert: {{ github.event_name == 'schedule' }} + fail-on-alert: ${{ github.event_name == 'schedule' }} - name: Report Latency results uses: benchmark-action/github-action-benchmark@v1.20.4 with: @@ -41,7 +41,7 @@ runs: tool: "customSmallerIsBetter" output-file-path: latency_results.json github-token: ${{ inputs.github-token }} - auto-push: {{ github.event_name == 'schedule' }} + auto-push: ${{ github.event_name == 'schedule' }} summary-always: true alert-threshold: "130%" - fail-on-alert: {{ github.event_name == 'schedule' }} \ No newline at end of file + fail-on-alert: ${{ github.event_name == 'schedule' }} \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a0af6ce26..ae7d042fd 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -47,13 +47,13 @@ jobs: # So I set the alert threshold to 130% of the previous benchmark result. # If the current benchmark result is more than 130% of the previous benchmark result, it will fail. alert-threshold: '130%' - fail-on-alert: {{ github.event_name == 'schedule' }} + fail-on-alert: ${{ github.event_name == 'schedule' }} alert-comment-cc-users: '@runwasi-committers' # Enable Job Summary summary-always: true # Automatically push the benchmark result to gh-pages branch # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details - auto-push: {{ github.event_name == 'schedule' }} + auto-push: ${{ github.event_name == 'schedule' }} benchmark-http: runs-on: ubuntu-latest From 4931652976d3d7b193cad89af25cb52d24f27ed6 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Mon, 16 Dec 2024 00:56:52 +0000 Subject: [PATCH 7/7] .github/actions/run-hey-load-test: Resolve non-fast-forward git fetch issue for unscheduled events Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/actions/run-hey-load-test/action.yml | 26 ++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index 3050b1200..a36085be3 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -35,13 +35,35 @@ runs: alert-threshold: "130%" fail-on-alert: ${{ github.event_name == 'schedule' }} - name: Report Latency results + if : ${{ github.event_name == 'schedule' }} + uses: benchmark-action/github-action-benchmark@v1.20.4 + with: + name: "HTTP Latency" + tool: "customSmallerIsBetter" + output-file-path: latency_results.json + github-token: ${{ inputs.github-token }} + auto-push: true + summary-always: true + alert-threshold: "130%" + fail-on-alert: true + + # If the event is not a schedule, we'd run into a problem of the failed `git fetch` command, + # which attempts to update the local `gh-pages` branch with changes from the remote repository + # but failed because the update is non-fast-forward. It failed because the previous step + # `benchmark-action/github-action-benchmark` has already committed the changes to the local, so + # it creates a conflict with the remote repository. + # + # The workaround is to use the `external-data-json-path` option to tell the action to not + # attempt to update the local `gh-pages` branch. + - name: Report Latency results + if : ${{ github.event_name != 'schedule' }} uses: benchmark-action/github-action-benchmark@v1.20.4 with: name: "HTTP Latency" tool: "customSmallerIsBetter" output-file-path: latency_results.json github-token: ${{ inputs.github-token }} - auto-push: ${{ github.event_name == 'schedule' }} summary-always: true alert-threshold: "130%" - fail-on-alert: ${{ github.event_name == 'schedule' }} \ No newline at end of file + fail-on-alert: false + external-data-json-path: ./cache/latency_results.json \ No newline at end of file