diff --git a/.github/actions/run-hey-load-test/action.yml b/.github/actions/run-hey-load-test/action.yml index 5364cb04b..a36085be3 100644 --- a/.github/actions/run-hey-load-test/action.yml +++ b/.github/actions/run-hey-load-test/action.yml @@ -20,27 +20,50 @@ runs: shell: bash run: | hey -n 100000 -c 50 http://127.0.0.1:8080 > raw-output.txt - python3 ./scripts/parse-hey.py raw-output.txt > benchmark-results.json - cat benchmark-results.json + python3 ./scripts/parse-hey.py raw-output.txt + cat latency_results.json + cat throughput_results.json - name: Report Throughput results uses: benchmark-action/github-action-benchmark@v1.20.4 with: name: "HTTP Throughput" tool: "customBiggerIsBetter" - output-file-path: benchmark-results.json + output-file-path: throughput_results.json github-token: ${{ inputs.github-token }} - external-data-json-path: ./cache/benchmark-data.json + auto-push: ${{ github.event_name == 'schedule' }} summary-always: true - alert-threshold: "120%" + alert-threshold: "130%" + fail-on-alert: ${{ github.event_name == 'schedule' }} + - name: Report Latency results + if : ${{ github.event_name == 'schedule' }} + uses: benchmark-action/github-action-benchmark@v1.20.4 + with: + name: "HTTP Latency" + tool: "customSmallerIsBetter" + output-file-path: latency_results.json + github-token: ${{ inputs.github-token }} + auto-push: true + summary-always: true + alert-threshold: "130%" fail-on-alert: true + + # If the event is not a schedule, we'd run into a problem of the failed `git fetch` command, + # which attempts to update the local `gh-pages` branch with changes from the remote repository + # but failed because the update is non-fast-forward. It failed because the previous step + # `benchmark-action/github-action-benchmark` has already committed the changes to the local, so + # it creates a conflict with the remote repository. + # + # The workaround is to use the `external-data-json-path` option to tell the action to not + # attempt to update the local `gh-pages` branch. - name: Report Latency results + if : ${{ github.event_name != 'schedule' }} uses: benchmark-action/github-action-benchmark@v1.20.4 with: name: "HTTP Latency" tool: "customSmallerIsBetter" - output-file-path: benchmark-results.json + output-file-path: latency_results.json github-token: ${{ inputs.github-token }} - external-data-json-path: ./cache/benchmark-data.json summary-always: true alert-threshold: "130%" - fail-on-alert: true \ No newline at end of file + fail-on-alert: false + external-data-json-path: ./cache/latency_results.json \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1fe67489c..ae7d042fd 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -5,6 +5,12 @@ on: - cron: '0 0 * * *' # Runs daily at midnight pull_request: +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + jobs: benchmark: runs-on: ubuntu-latest @@ -24,18 +30,12 @@ jobs: - name: Build and load shims and wasi-demo-app shell: bash run: | - make build install test-image load test-image/oci load/oci test-image/http load/http + make build install test-image load test-image/oci load/oci - name: Run Benchmarks shell: bash run: | set -o pipefail cargo bench -p containerd-shim-benchmarks -- --output-format bencher | tee output.txt - # Download previous benchmark result from cache (if exists) - - name: Cache benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1.20.4 with: @@ -47,12 +47,33 @@ jobs: # So I set the alert threshold to 130% of the previous benchmark result. # If the current benchmark result is more than 130% of the previous benchmark result, it will fail. alert-threshold: '130%' - fail-on-alert: true + fail-on-alert: ${{ github.event_name == 'schedule' }} alert-comment-cc-users: '@runwasi-committers' # Enable Job Summary summary-always: true - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json + # Automatically push the benchmark result to gh-pages branch + # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details + auto-push: ${{ github.event_name == 'schedule' }} + + benchmark-http: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Fetch submodules + run: git submodule update --init --recursive + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile + - name: Setup build environment + shell: bash + run: | + os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') + ./scripts/setup-$os.sh + - name: Build and load shims and wasi-demo-app + shell: bash + run: | + make build install test-image/http load/http - name: Start wasmtime shim shell: bash run: | @@ -73,4 +94,4 @@ jobs: if: success() shell: bash run: | - sudo ctr task kill -s SIGKILL wasi-http \ No newline at end of file + sudo ctr task kill -s SIGKILL wasi-http diff --git a/scripts/parse-hey.py b/scripts/parse-hey.py index d946441da..438c166d6 100644 --- a/scripts/parse-hey.py +++ b/scripts/parse-hey.py @@ -57,19 +57,20 @@ def parse_hey_output(file_path): latency = latency * 1000 if latency is not None else None - results = [] if rps is not None: - results.append({ + throughput_result = [{ "name": "HTTP RPS", "unit": "req/s", "value": rps - }) + }] + with open('throughput_results.json', 'w') as f: + json.dump(throughput_result, f, indent=2) if latency is not None: - results.append({ + latency_result = [{ "name": "HTTP p95 Latency", "unit": "ms", "value": latency - }) - - print(json.dumps(results, indent=2)) + }] + with open('latency_results.json', 'w') as f: + json.dump(latency_result, f, indent=2)