benchmarks.yml

name: "OpenVM Benchmarks: Coordinate Runner & Reporting"

on:
  push:
    branches: ["main"]
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    branches: ["**"]
    paths:
      - "crates/circuits/**"
      - "crates/vm/**"
      - "crates/toolchain/**"
      - "crates/extensions/**"
      - "benchmarks/**"
      - ".github/workflows/benchmark-call.yml"
      - ".github/workflows/benchmarks.yml"

concurrency:
  group: benchmark-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  OPENVM_FAST_TEST: "1"
  CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
  CARGO_NET_GIT_FETCH_WITH_CLI: "true"

permissions:
  contents: write
  pull-requests: write

jobs:
  create-matrix:
    runs-on:
      - runs-on=${{ github.run_id }}
      - runner=8cpu-linux-x64
    outputs:
      matrix: ${{ steps.create-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref || github.ref }}

      - name: Create benchmark matrix from JSON
        id: create-matrix
        run: |
          if [ ! -f ./ci/benchmark-config.json ]; then
            echo "Error: ./ci/benchmark-config.json not found"
            exit 1
          fi
          cat ./ci/benchmark-config.json

          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            LABELS='${{ toJson(github.event.pull_request.labels) }}'
            RUN_E2E=$(echo "$LABELS" | jq 'any(.name == "run-benchmark-e2e")')
          else
            RUN_E2E=false
          fi

          matrix=$(jq -c --argjson run_e2e $RUN_E2E '
            [
              .benchmarks[] |
              .name as $name |
              .id as $id |
              .e2e_bench as $e2e_bench |
              select($run_e2e or .e2e_bench != true) |
              .run_params[] |
              {
                name: $name,
                e2e_bench: $e2e_bench,
                id: $id,
                instance_type: .instance_type,
                memory_allocator: .memory_allocator,
                app_log_blowup: .app_log_blowup,
                agg_log_blowup: .agg_log_blowup,
                root_log_blowup: (.root_log_blowup // 0),
                internal_log_blowup: (.internal_log_blowup // 0),
                max_segment_length: (.max_segment_length // 1048476)
              }
            ]
          ' ./ci/benchmark-config.json)
          if [ $? -ne 0 ]; then
            echo "Error: Failed to parse ./ci/benchmark-config.json"
            exit 1
          fi
          echo "matrix=$matrix" >> $GITHUB_OUTPUT

  benchmark:
    needs: create-matrix
    strategy:
      matrix:
        benchmark_run: ${{fromJson(needs.create-matrix.outputs.matrix)}}
    uses: ./.github/workflows/benchmark-call.yml
    with:
      benchmark_name: ${{ matrix.benchmark_run.name }}
      benchmark_id: ${{ matrix.benchmark_run.id }}
      instance_type: ${{ matrix.benchmark_run.instance_type }}
      memory_allocator: ${{ matrix.benchmark_run.memory_allocator }}
      app_log_blowup: ${{ matrix.benchmark_run.app_log_blowup }}
      agg_log_blowup: ${{ matrix.benchmark_run.agg_log_blowup }}
      root_log_blowup: ${{ matrix.benchmark_run.root_log_blowup }}
      internal_log_blowup: ${{ matrix.benchmark_run.internal_log_blowup }}
      max_segment_length: ${{ matrix.benchmark_run.max_segment_length }}
      e2e_bench: ${{ matrix.benchmark_run.e2e_bench }}
    secrets: inherit

  summarize:
    needs: [create-matrix, benchmark]
    runs-on:
      - runs-on=${{ github.run_id }}
      - runner=8cpu-linux-arm64
    steps:
      ##########################################################################
      # Install S3 if necessary                                                #
      ##########################################################################
      - name: Install architecture specific tools
        run: |
          S5CMD_BIN="s5cmd_2.2.2_linux_arm64.deb"
          echo "Checking s5cmd"
          if type s5cmd &>/dev/null; then
              echo "s5cmd was installed."
          else
              TMP_DIR=/tmp/s5cmd
              rm -rf $TMP_DIR
              mkdir $TMP_DIR
              echo "s5cmd was not installed. Installing.."
              wget "https://github.com/peak/s5cmd/releases/download/v2.2.2/${S5CMD_BIN}" -P $TMP_DIR
              sudo dpkg -i "${TMP_DIR}/${S5CMD_BIN}"
          fi

      ##########################################################################
      # Download individual result .md files from S3 and combine them          #
      ##########################################################################
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref || github.ref }}

      - name: Set github pages path for PR
        if: github.event_name == 'pull_request'
        run: |
          BENCHMARK_RESULTS_PATH="benchmarks-pr/${{ github.event.pull_request.number }}"
          echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV

      - name: Set github pages path for push
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          BENCHMARK_RESULTS_PATH="benchmarks"
          echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV

      - name: Load all metadata files from S3
        run: |
          current_sha=$(git rev-parse HEAD)
          md_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r '
            .[] |
            select(.e2e_bench != true) |
            "\(.id)-"' |
            sed "s/$/${current_sha}.md/" |
            sort)
          md_file_list=$(echo -n "$md_files" | paste -sd "," -)

          e2e_md_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r '
            .[] |
            select(.e2e_bench == true) |
            "\(.id)-"' |
            sed "s/$/${current_sha}.md/" |
            sort)
          e2e_md_file_list=$(echo -n "$e2e_md_files" | paste -sd "," -)

          while read md_file; do
            if [ -z "$md_file" ]; then
              continue
            fi
            echo "Downloading results for benchmark: $md_file"
            s5cmd cp "s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results/${md_file}" "${md_file}"
          done <<< "$md_files"

          E2E_FILE_LIST=""
          if [[ -n "$e2e_md_files" ]]; then
            while read e2e_md_file; do
              echo "Downloading results for benchmark: $e2e_md_file"
              s5cmd cp "s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results/${e2e_md_file}" "${e2e_md_file}"
            done <<< "$e2e_md_files"
            E2E_FILE_LIST="${e2e_md_file_list}"
          fi

          echo "Benchmarks: ${md_file_list}"
          echo "E2E Benchmarks: ${E2E_FILE_LIST}"

          python3 ci/scripts/metric_unify/summarize.py "${md_file_list}" \
            --e2e-md-files "${E2E_FILE_LIST}" \
            --aggregation-json ci/scripts/metric_unify/aggregation.json \
            --benchmark-results-link "https://github.com/${{ github.repository }}/blob/benchmark-results/${BENCHMARK_RESULTS_PATH}"

          echo "" >> summary.md
          echo "Commit: https://github.com/${{ github.repository }}/commit/${CURRENT_SHA}" >> summary.md
          echo "" >> summary.md
          echo "[Benchmark Workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md
          mkdir -p /tmp/benchmark-results/
          cp summary.md /tmp/benchmark-results/

      ##########################################################################
      # Update benchmark-results with summary upon a PR event                           #
      ##########################################################################
      - uses: actions/checkout@v4
        if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        with:
          ref: benchmark-results

      - name: Set up git
        if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        run: |
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "github-actions[bot]"

      - name: Update github pages with new bench results
        if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
        run: |
          mkdir -p ${BENCHMARK_RESULTS_PATH}
          cp /tmp/benchmark-results/summary.md ${BENCHMARK_RESULTS_PATH}/summary.md
          git add ${BENCHMARK_RESULTS_PATH}/summary.md
          if [[ "${{ github.event_name }}" == "push" ]]; then
            cp /tmp/benchmark-results/summary.md index.md
            git add index.md
          fi
          git commit --allow-empty -m "Update summarized benchmark result at ${BENCHMARK_RESULTS_PATH}/summary.md"
          git push --force

      ##########################################################################
      # Update PR comment upon a pull request event                            #
      ##########################################################################
      - name: Collapse previous comment (if exists)
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const comments = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number
            });
            for (const comment of comments.data) {
              if (comment.user.login == "github-actions[bot]" && comment.body.startsWith("<!--Benchmark Results-->")) {
                console.log("collapse comment ", comment.id);
                const resp = await github.graphql(`
                  mutation {
                    minimizeComment(input: {classifier: OUTDATED, subjectId: "${comment.node_id}"}) {
                      minimizedComment {
                        isMinimized
                      }
                    }
                  }
                `);
              }
            }

      - name: Add comment to pull request
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs')
            const newBenchmark = fs.readFileSync('/tmp/benchmark-results/summary.md', { encoding: 'utf8', flag: 'r' })

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `<!--Benchmark Results-->\n${newBenchmark}`
            });