fix: generate plots alt #62
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run benchmark" | |
| on: | |
| pull_request_target: | |
| types: [assigned, opened, synchronize, reopened, edited] | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| build: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Set up Docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v6 | |
| env: | |
| DOCKER_BUILD_RECORD_UPLOAD: false | |
| with: | |
| push: true | |
| tags: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| benchmark: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: build | |
| runs-on: ubuntu-latest | |
| # runs-on: | |
| # group: benchmarking-runner | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| matrix: | |
| response: [big, medium, small] | |
| service: [ | |
| # the theoretical maximum | |
| # source_graphql, | |
| # source_rest_api, | |
| # nginx -> source: the baseline for comparison | |
| nginx_graphql, | |
| # nginx_rest_api, | |
| # tailcall configurations | |
| tailcall_default, | |
| # tailcall_tweaks, | |
| # tailcall_http_cache, | |
| # tailcall_cache_dir, | |
| # tailcall_dedupe_op, | |
| tailcall_full_conf, | |
| # wundergraph configurations | |
| wundergraph_no_opt, | |
| # wundergraph_dedupe, | |
| wundergraph_default, | |
| # apollo | |
| apollo_router, | |
| # grafbase | |
| grafbase_default, | |
| grafbase_cache, | |
| ] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create "output.log" file | |
| run: touch ${{ github.workspace }}/output.log | |
| - id: benchmark | |
| name: Benchmark | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| registry: ghcr.io | |
| image: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| options: -v ${{ github.workspace }}:/work | |
| run: ./benchmark.sh ${{ matrix.response }} ${{ matrix.service }} >> /work/output.log | |
| - name: Print result | |
| run: cat ${{ github.workspace }}/output.log | |
| - name: Determine if run had errors | |
| id: extract_has_error | |
| run: | | |
| HAS_ERROR=$((grep -q "Error distribution:" ${{ github.workspace }}/output.log && echo true || echo false) | sed 's/true/❌/; s/false/✅/') | |
| echo "error=${HAS_ERROR}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Extract RPS | |
| id: extract_rps | |
| run: | | |
| RPS=$(grep "Requests/sec:" ${{ github.workspace }}/output.log | awk '{printf int($2)}' | sed -E ':a;s/([0-9])([0-9]{3})$/\1,\2/;ta') | |
| echo "rps=${RPS}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Extract P95 Latency | |
| id: extract_p95 | |
| run: | | |
| P95=$(grep "95%" ${{ github.workspace }}/output.log | awk '{print $3}') | |
| echo "p95=${P95}" | tee -a "$GITHUB_OUTPUT" | |
| - uses: cloudposse/github-action-matrix-outputs-write@v1 | |
| id: out | |
| with: | |
| matrix-step-name: ${{ github.job }} | |
| matrix-key: ${{ matrix.response }}_${{ matrix.service }} | |
| outputs: |- | |
| has_error: ${{ steps.extract_has_error.outputs.error }} | |
| rps: ${{ steps.extract_rps.outputs.rps }} | |
| p95: ${{ steps.extract_p95.outputs.p95 }} | |
| read: | |
| runs-on: ubuntu-latest | |
| needs: [benchmark] | |
| steps: | |
| - uses: cloudposse/github-action-matrix-outputs-read@v1 | |
| id: read | |
| with: | |
| matrix-step-name: benchmark | |
| outputs: | |
| result: "${{ steps.read.outputs.result }}" | |
| analyze: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Install gnuplot | |
| run: sudo apt-get update && sudo apt-get install -y gnuplot | |
| - name: Collect results table | |
| run: | | |
| echo "### [Small Payload - 362 bytes](./source/small.json)" >> "results.md" | |
| echo "| Server | Status | RPS | Latency |" >> "results.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_nginx_graphql }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_nginx_graphql }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_nginx_graphql }} sec\` |" >> "results.md" | |
| echo "| **Base** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_tailcall_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_tailcall_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_tailcall_default }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_grafbase_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_grafbase_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_grafbase_default }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_wundergraph_no_opt }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_wundergraph_no_opt }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_wundergraph_no_opt }} sec\` |" >> "results.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_apollo_router }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_apollo_router }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_apollo_router }} sec\` |" >> "results.md" | |
| echo "| **Cached** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_tailcall_full_conf }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_tailcall_full_conf }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_tailcall_full_conf }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_grafbase_cache }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_grafbase_cache }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_grafbase_cache }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.small_wundergraph_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.small_wundergraph_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.small_wundergraph_default }} sec\` |" >> "results.md" | |
| echo "### [Medium Payload - 12,598 bytes](./source/medium.json)" >> "results.md" | |
| echo "| Server | Status | RPS | Latency |" >> "results.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_nginx_graphql }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_nginx_graphql }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_nginx_graphql }} sec\` |" >> "results.md" | |
| echo "| **Base** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_tailcall_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_tailcall_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_tailcall_default }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_grafbase_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_grafbase_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_grafbase_default }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_wundergraph_no_opt }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_wundergraph_no_opt }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_wundergraph_no_opt }} sec\` |" >> "results.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_apollo_router }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_apollo_router }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_apollo_router }} sec\` |" >> "results.md" | |
| echo "| **Cached** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_tailcall_full_conf }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_tailcall_full_conf }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_tailcall_full_conf }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_grafbase_cache }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_grafbase_cache }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_grafbase_cache }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.medium_wundergraph_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.medium_wundergraph_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.medium_wundergraph_default }} sec\` |" >> "results.md" | |
| echo "### [Big Payload - 112,838 bytes](./source/big.json)" >> "results.md" | |
| echo "| Server | Status | RPS | Latency |" >> "results.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_nginx_graphql }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_nginx_graphql }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_nginx_graphql }} sec\` |" >> "results.md" | |
| echo "| **Base** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_tailcall_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_tailcall_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_tailcall_default }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_grafbase_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_grafbase_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_grafbase_default }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_wundergraph_no_opt }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_wundergraph_no_opt }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_wundergraph_no_opt }} sec\` |" >> "results.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_apollo_router }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_apollo_router }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_apollo_router }} sec\` |" >> "results.md" | |
| echo "| **Cached** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_tailcall_full_conf }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_tailcall_full_conf }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_tailcall_full_conf }} sec\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_grafbase_cache }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_grafbase_cache }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_grafbase_cache }} sec\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \ | |
| ${{ fromJson(needs.read.outputs.result).has_error.big_wundergraph_default }} | \ | |
| \`${{ fromJson(needs.read.outputs.result).rps.big_wundergraph_default }} RPS\` | \ | |
| \`${{ fromJson(needs.read.outputs.result).p95.big_wundergraph_default }} sec\` |" >> "results.md" | |
| - name: Update `README.md` file with table data | |
| run: | | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/,/<!-- PERFORMANCE_RESULTS_END -->/ { | |
| /<!-- PERFORMANCE_RESULTS_START -->/!{ | |
| /<!-- PERFORMANCE_RESULTS_END -->/!d | |
| } | |
| }' README.md | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/r results.md' README.md | |
| - name: Summarize | |
| run: | | |
| echo "### Results" >> "$GITHUB_STEP_SUMMARY" | |
| cat "results.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Write plot script | |
| run: | | |
| cat << 'EOF' > plot_script.gp | |
| set terminal png size 800,500 enhanced font "Arial,20" | |
| set output ARG1 | |
| small = "#D81B60"; medium = "#1E88E5"; big = "#FFC107"; | |
| set yrange [0:20] | |
| set style data histogram | |
| set style histogram cluster gap 1 | |
| set style fill solid | |
| set boxwidth 0.9 | |
| set xtics format "" | |
| set grid ytics | |
| set title ARG2 | |
| plot 'data.dat' using 2:xtic(1) title "Small" linecolor rgb small, \ | |
| 'data.dat' using 3 title "Medium" linecolor rgb medium, \ | |
| 'data.dat' using 4 title "Big" linecolor rgb big | |
| EOF | |
| - name: Save RPS basic plot data | |
| run: | | |
| cat << 'EOF' > data.dat | |
| # Small Medium Big | |
| Tailcall ${{ fromJson(needs.read.outputs.result).rps.small_tailcall_default }} ${{ fromJson(needs.read.outputs.result).rps.medium_tailcall_default }} ${{ fromJson(needs.read.outputs.result).rps.big_tailcall_default }} | |
| Grafbase ${{ fromJson(needs.read.outputs.result).rps.small_grafbase_default }} ${{ fromJson(needs.read.outputs.result).rps.medium_grafbase_default }} ${{ fromJson(needs.read.outputs.result).rps.big_grafbase_default }} | |
| Wundergraph ${{ fromJson(needs.read.outputs.result).rps.small_wundergraph_no_opt }} ${{ fromJson(needs.read.outputs.result).rps.medium_wundergraph_no_opt }} ${{ fromJson(needs.read.outputs.result).rps.big_wundergraph_no_opt }} | |
| Apollo ${{ fromJson(needs.read.outputs.result).rps.small_apollo_router }} ${{ fromJson(needs.read.outputs.result).rps.medium_apollo_router }} ${{ fromJson(needs.read.outputs.result).rps.big_apollo_router }} | |
| EOF | |
| - name: Create RPS basic plot | |
| run: gnuplot -c "files/rps_default.png" "RPS Default" | |
| - name: Cleanup files | |
| run: | | |
| rm "results.md" | |
| rm "data.dat" | |
| rm "plot_script.gp" | |
| - name: Comment `README.md` file | |
| if: github.event_name == 'pull_request_target' | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| sha: ${{ github.event.pull_request.head.sha }} | |
| body-path: "README.md" | |
| reactions: eyes | |
| - name: Commit and push changes (on main branch) | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: main | |
| commit_author: Author <[email protected]> | |
| commit_message: "[ci skip] update performance results in README.md" |