benchmark-call.yml

name: "Benchmark: Coordinate Runner & Reporting"

on:
  workflow_dispatch:
    inputs:
      benchmark_name:
        type: choice
        required: true
        description: The name of the benchmark to run
        options:
          - verify_fibair
          - fibonacci
          - alu256_e2e
          - small_e2e
      instance_type:
        type: string
        required: false
        description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-arm64)
        default: 64cpu-linux-arm64
  workflow_call:
    inputs:
      benchmark_name:
        type: string
        required: true
        description: The name of the benchmark to run
      instance_type:
        type: string
        required: false
        description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-arm64)
        default: 64cpu-linux-arm64

env:
  S3_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/results
  S3_METRICS_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/metrics
  PUBLIC_S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
  FEATURE_FLAGS: "bench-metrics,parallel,mimalloc"
  CMD_ARGS: ""

jobs:
  bench-new:
    name: Run benchmark on workflow ref/branch
    runs-on:
      - runs-on
      - runner=${{ inputs.instance_type }}
      - image=arm64-rust-dev-32gb
      - run-id=${{ github.run_id }}
      - tag=bench-${{ inputs.benchmark_name }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref || github.ref }}
      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install rust src
        run: |
          rustup component add rust-src --toolchain nightly-aarch64-unknown-linux-gnu

      - name: Feature flags
        if: contains(github.event.pull_request.labels.*.name, 'run-benchmark') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        run: |
          echo "Adding aggregation feature flag"
          echo "FEATURE_FLAGS=${FEATURE_FLAGS},aggregation" >> $GITHUB_ENV

      - name: Setup halo2
        working-directory: lib/recursion # We only ever run halo2 for recursion
        run: |
          if [[ "${{ github.event.pull_request.labels[*].name }}" == *"run-benchmark-e2e"* ]]; then
            echo "Adding static-verifier feature flag"
            echo "FEATURE_FLAGS=${FEATURE_FLAGS},static-verifier" >> $GITHUB_ENV
            bash trusted_setup_s3.sh
          fi

      - name: Set BIN_NAME and CMD_ARGS
        run: |
          CMD_ARGS="--features ${FEATURE_FLAGS}"
          echo "CMD_ARGS=${CMD_ARGS}" >> $GITHUB_ENV
          echo "BIN_NAME=${{ inputs.benchmark_name }}" >> $GITHUB_ENV

      ######################################################
      # Run a different benchmark based on benchmark_name:
      - name: Run benchmark
        if: inputs.benchmark_name == 'verify_fibair'
        working-directory: lib/recursion
        run: |
          python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS

      - name: Run benchmark
        if: inputs.benchmark_name == 'fibonacci'
        working-directory: benchmarks
        run: |
          python3 ../ci/scripts/bench.py $BIN_NAME $CMD_ARGS

      - name: Run benchmark
        if: inputs.benchmark_name == 'alu256_e2e'
        working-directory: lib/recursion
        run: |
          python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS

      - name: Run benchmark
        if: inputs.benchmark_name == 'small_e2e'
        working-directory: lib/recursion
        run: |
          python3 ../../ci/scripts/bench.py $BIN_NAME $CMD_ARGS

      ######################################################
      - name: Store metric json and compute diff with previous
        run: |
          METRIC_PATH=".bench_metrics/${BIN_NAME}.json"
          echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV

          current_sha=$(git rev-parse HEAD)
          echo "Current SHA: $current_sha"
          echo "current_sha=${current_sha}" >> $GITHUB_ENV

          if [[ -f $METRIC_PATH ]]; then
            s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${current_sha}-${{ inputs.benchmark_name }}.json

            prev_path="${{ env.S3_METRICS_PATH }}/main-${{ inputs.benchmark_name }}.json"
            count=`s5cmd ls $prev_path | wc -l`

            if [[ $count -gt 0 ]]; then
              s5cmd cp $prev_path prev.json
              python3 ci/scripts/metric_unify/main.py $METRIC_PATH --prev prev.json --aggregation-json ci/scripts/metric_unify/aggregation.json > results.md
            else
              echo "No previous benchmark on main branch found"
              python3 ci/scripts/metric_unify/main.py $METRIC_PATH --aggregation-json ci/scripts/metric_unify/aggregation.json > results.md
            fi
          else
            echo "No benchmark metrics found at ${METRIC_PATH}"
          fi

      - name: Install inferno-flamegraph
        run: cargo install inferno

      - name: Generate flamegraphs
        run: |
          if [[ -f $METRIC_PATH ]]; then
            python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
            s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/"
            echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
          fi

      - name: Add benchmark metadata
        run: |
          commit_url="https://github.com/${{ github.repository }}/commit/${current_sha}"
          RESULT_PATH=results.md
          echo "" >> $RESULT_PATH
          if [[ "$UPLOAD_FLAMEGRAPHS" == '1' ]]; then
            echo "<details>" >> $RESULT_PATH
            echo "<summary>Flamegraphs</summary>" >> $RESULT_PATH
            echo "" >> $RESULT_PATH
            for file in .bench_metrics/flamegraphs/*.svg; do
              filename=$(basename "$file")
              flamegraph_url=https://axiom-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs/${current_sha}/${filename}
              echo "[![]($flamegraph_url)]($flamegraph_url)" >> $RESULT_PATH
            done
            echo "" >> $RESULT_PATH
            echo "</details>" >> $RESULT_PATH
            echo "" >> $RESULT_PATH
          fi
          echo "Commit: ${commit_url}" >> $RESULT_PATH
          echo "Instance Type: ${{ inputs.instance_type }}" >> $RESULT_PATH
          echo "[Benchmark Workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $RESULT_PATH
          s5cmd cp $RESULT_PATH "${{ env.S3_PATH }}/${current_sha}-${{ inputs.benchmark_name }}.md"

      - name: Collapse previous comment (if exists)
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const comments = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number
            });
            for (const comment of comments.data) {
              if (comment.user.login == "github-actions[bot]" && comment.body.startsWith("<!--${{ inputs.benchmark_name }}-->")) {
                console.log("collapse comment ", comment.id);
                const resp = await github.graphql(`
                  mutation {
                    minimizeComment(input: {classifier: OUTDATED, subjectId: "${comment.node_id}"}) {
                      minimizedComment {
                        isMinimized
                      }
                    }
                  }
                `);
              }
            }

      - name: Add comment to pull request
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs')
            const newBenchmark = fs.readFileSync('./results.md', { encoding: 'utf8', flag: 'r' })

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `<!--${{ inputs.benchmark_name }}-->\n${newBenchmark}`
            });

      - name: Update latest main result in s3
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          s5cmd cp "${{ env.S3_PATH }}/${{ env.current_sha }}-${{ inputs.benchmark_name }}.md" "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md"

          if [[ -f $METRIC_PATH ]]; then
            s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/main-${{ inputs.benchmark_name }}.json"
          fi

      - uses: actions/checkout@v4
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          ref: gh-pages

      - name: set up git
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "github-actions[bot]"

      - name: Update github pages with new bench results
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          s5cmd cp "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md" benchmarks/${{ inputs.benchmark_name }}.md
          git add benchmarks/${{ inputs.benchmark_name }}.md
          git commit --allow-empty -m "Update benchmark result for ${{ inputs.benchmark_name }}"
          git push