fix: remove colors #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run benchmark" | |
| on: | |
| pull_request_target: | |
| types: [assigned, opened, synchronize, reopened, edited] | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and Push Docker Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| tags: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| benchmark: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| # runs-on: | |
| # group: benchmark-runner | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| matrix: | |
| response: [big, medium, small] | |
| service: [ | |
| # the theoretical maximum | |
| # source_graphql, | |
| # source_rest_api, | |
| # nginx -> source: the baseline for comparison | |
| nginx_graphql, | |
| # nginx_rest_api, | |
| # tailcall configurations | |
| tailcall_default, | |
| # tailcall_tweaks, | |
| # tailcall_http_cache, | |
| # tailcall_cache_dir, | |
| # tailcall_dedupe_op, | |
| tailcall_full_conf, | |
| # wundergraph configurations | |
| wundergraph_no_opt, | |
| # wundergraph_dedupe, | |
| wundergraph_default, | |
| # apollo | |
| apollo_router, | |
| # grafbase | |
| grafbase_default, | |
| grafbase_cache, | |
| ] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create "output.log" file | |
| run: touch ${{ github.workspace }}/output.log | |
| - id: benchmark | |
| name: Benchmark | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| registry: ghcr.io | |
| image: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| options: -v ${{ github.workspace }}:/work | |
| run: ./docker_benchmark.sh ${{ matrix.response }} ${{ matrix.service }} >> /work/output.log | |
| - name: Print Result | |
| run: cat ${{ github.workspace }}/output.log | |
| - name: Extract Requests Per Second | |
| id: extract_rps | |
| run: | | |
| RPS=$(grep "Requests/sec:" ${{ github.workspace }}/output.log | awk '{printf int($2)}') | |
| echo "${{ matrix.response }}_${{ matrix.service }}_rps=${RPS}" | tee -a "$GITHUB_OUTPUT" | |
| outputs: | |
| big_nginx_graphql: ${{ steps.extract_rps.outputs.big_nginx_graphql_rps }} | |
| medium_nginx_graphql: ${{ steps.extract_rps.outputs.medium_nginx_graphql_rps }} | |
| small_nginx_graphql: ${{ steps.extract_rps.outputs.small_nginx_graphql_rps }} | |
| big_tailcall_default: ${{ steps.extract_rps.outputs.big_tailcall_default_rps }} | |
| medium_tailcall_default: ${{ steps.extract_rps.outputs.medium_tailcall_default_rps }} | |
| small_tailcall_default: ${{ steps.extract_rps.outputs.small_tailcall_default_rps }} | |
| big_grafbase_default: ${{ steps.extract_rps.outputs.big_grafbase_default_rps }} | |
| medium_grafbase_default: ${{ steps.extract_rps.outputs.medium_grafbase_default_rps }} | |
| small_grafbase_default: ${{ steps.extract_rps.outputs.small_grafbase_default_rps }} | |
| big_wundergraph_no_opt: ${{ steps.extract_rps.outputs.big_wundergraph_no_opt_rps }} | |
| medium_wundergraph_no_opt: ${{ steps.extract_rps.outputs.medium_wundergraph_no_opt_rps }} | |
| small_wundergraph_no_opt: ${{ steps.extract_rps.outputs.small_wundergraph_no_opt_rps }} | |
| big_apollo_router: ${{ steps.extract_rps.outputs.big_apollo_router_rps }} | |
| medium_apollo_router: ${{ steps.extract_rps.outputs.medium_apollo_router_rps }} | |
| small_apollo_router: ${{ steps.extract_rps.outputs.small_apollo_router_rps }} | |
| big_tailcall_full_conf: ${{ steps.extract_rps.outputs.big_tailcall_full_conf_rps }} | |
| medium_tailcall_full_conf: ${{ steps.extract_rps.outputs.medium_tailcall_full_conf_rps }} | |
| small_tailcall_full_conf: ${{ steps.extract_rps.outputs.small_tailcall_full_conf_rps }} | |
| big_grafbase_cache: ${{ steps.extract_rps.outputs.big_grafbase_cache_rps }} | |
| medium_grafbase_cache: ${{ steps.extract_rps.outputs.medium_grafbase_cache_rps }} | |
| small_grafbase_cache: ${{ steps.extract_rps.outputs.small_grafbase_cache_rps }} | |
| big_wundergraph_default: ${{ steps.extract_rps.outputs.big_wundergraph_default_rps }} | |
| medium_wundergraph_default: ${{ steps.extract_rps.outputs.medium_wundergraph_default_rps }} | |
| small_wundergraph_default: ${{ steps.extract_rps.outputs.small_wundergraph_default_rps }} | |
| analyze: | |
| needs: benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install `termgraph` | |
| run: python -m pip install termgraph | |
| - id: collect | |
| name: Collect Requests Per Second from all matrix jobs | |
| run: | | |
| echo "| Server | 112,838 bytes | 12598 bytes | 362 bytes |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| --- | --- | --- | --- |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Nginx Default | ${{ needs.benchmark.outputs.big_nginx_graphql }} | ${{ needs.benchmark.outputs.medium_nginx_graphql }} | ${{ needs.benchmark.outputs.small_nginx_graphql }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Base | --- | --- | --- |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Tailcall | ${{ needs.benchmark.outputs.big_tailcall_default }} | ${{ needs.benchmark.outputs.medium_tailcall_default }} | ${{ needs.benchmark.outputs.small_tailcall_default }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Grafbase | ${{ needs.benchmark.outputs.big_grafbase_default }} | ${{ needs.benchmark.outputs.medium_grafbase_default }} | ${{ needs.benchmark.outputs.small_grafbase_default }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Wundegraph | ${{ needs.benchmark.outputs.big_wundergraph_no_opt }} | ${{ needs.benchmark.outputs.medium_wundergraph_no_opt }} | ${{ needs.benchmark.outputs.small_wundergraph_no_opt }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Apollo | ${{ needs.benchmark.outputs.big_apollo_router }} | ${{ needs.benchmark.outputs.medium_apollo_router }} | ${{ needs.benchmark.outputs.small_apollo_router }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Cached | --- | --- | --- |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Tailcall | ${{ needs.benchmark.outputs.big_tailcall_full_conf }} | ${{ needs.benchmark.outputs.medium_tailcall_full_conf }} | ${{ needs.benchmark.outputs.small_tailcall_full_conf }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Grafbase Cache | ${{ needs.benchmark.outputs.big_grafbase_cache }} | ${{ needs.benchmark.outputs.medium_grafbase_cache }} | ${{ needs.benchmark.outputs.small_grafbase_cache }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "| Wundegraph Ludicrous | ${{ needs.benchmark.outputs.big_wundergraph_default }} | ${{ needs.benchmark.outputs.medium_wundergraph_default }} | ${{ needs.benchmark.outputs.small_wundergraph_default }} |" | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Prepare Plot (Basic) Data | |
| run: | | |
| echo -e "@ Big,Medium,Small\n" \ | |
| "Tailcall,${{ needs.benchmark.outputs.big_tailcall_default }},${{ needs.benchmark.outputs.medium_tailcall_default }},${{ needs.benchmark.outputs.small_tailcall_default }}\n" \ | |
| "Grafbase,${{ needs.benchmark.outputs.big_grafbase_default }},${{ needs.benchmark.outputs.medium_grafbase_default }},${{ needs.benchmark.outputs.small_grafbase_default }}\n" \ | |
| "Wundegraph,${{ needs.benchmark.outputs.big_wundergraph_no_opt }},${{ needs.benchmark.outputs.medium_wundergraph_no_opt }},${{ needs.benchmark.outputs.small_wundergraph_no_opt }}\n" \ | |
| "Apollo,${{ needs.benchmark.outputs.big_apollo_router }},${{ needs.benchmark.outputs.medium_apollo_router }},${{ needs.benchmark.outputs.small_apollo_router }}" | tee -a "plot1.dat" | |
| - name: Prepare Plot (Cached) Data | |
| run: | | |
| echo -e "@ Big,Medium,Small\n" \ | |
| "Tailcall,${{ needs.benchmark.outputs.big_tailcall_full_conf }},${{ needs.benchmark.outputs.medium_tailcall_full_conf }},${{ needs.benchmark.outputs.small_tailcall_full_conf }}\n" \ | |
| "Grafbase Cache,${{ needs.benchmark.outputs.big_grafbase_cache }},${{ needs.benchmark.outputs.medium_grafbase_cache }},${{ needs.benchmark.outputs.small_grafbase_cache }}\n" \ | |
| "Wundegraph Ludicrous,${{ needs.benchmark.outputs.big_wundergraph_default }},${{ needs.benchmark.outputs.medium_wundergraph_default }},${{ needs.benchmark.outputs.small_wundergraph_default }}\n" | tee -a "plot2.dat" | |
| - name: Plot (Basic) Data | |
| run: termgraph plot1.dat | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Plot (Cached) Data | |
| run: termgraph plot2.dat | tee -a "$GITHUB_STEP_SUMMARY" |