bench_matrix.yml

# Run all benchmarks when a comment containing "test performance please"
# comment is posted on a PR.

name: Benchmarks Matrix

on:
  issue_comment:
    types: [created]
  workflow_dispatch:
    inputs:
      commit:
        description: "Commit to benchmark."
        required: true
        type: string
      repo:
        description: "GitHub repository containing the commit to benchmark."
        required: true
        type: string
        default: "mbovel/scala3" # TODO(mbovel): Change to scala/scala3
      runs:
        description: "Number of runs to perform."
        required: true
        type: number
        default: 1
      profile:
        description: "Profile to run: 'merge' to run the benchmarks that are run on every PR merge (shorter), 'nightly' to run nightly benchmarks (longer), or 'all' to run both."
        required: true
        type: choice
        options:
          - merge
          - nightly
          - all

jobs:
  start_comment:
    name: Start comment
    if: github.event_name == 'issue_comment'
    runs-on: ubuntu-latest
    steps:
      - name: Comment
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: `Thanks for your request. Your benchmarks will be run shortly.\nYou can follow the progress at https://github.com/mbovel/dotty/actions/runs/${context.runId}.`
            })

  generate_runs:
    name: Generate run definitions
    if: github.event_name == 'workflow_dispatch' || (github.event.issue.pull_request && contains(github.event.comment.body, 'test performance please'))
    runs-on: ['self-hosted', 'benchmarks']
    env:
      # Path to the directory where the benchmark data is stored on the runner.
      # Keep in sync with the value in .github/workflows/bench.yml.
      DATA_DIR: /home/scalabenchs/bench-data-v3
    steps:
      - id: generate_runs
        uses: actions/github-script@v7
        with:
          script: |
            async function get_commits() {
              if (context.eventName === 'issue_comment') {
                const {data} = await github.rest.pulls.get({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  pull_number: context.payload.issue.number
                });
                const base = {
                  commit: data.base.sha,
                  repo: data.base.repo.full_name
                };
                core.setOutput('base', base.commit);
                const head = {
                  commit: data.head.sha,
                  repo: data.head.repo.full_name
                };
                core.setOutput('head', head.commit);
                return [base, head];
              } else if (context.eventName === 'workflow_dispatch') {
                return [{
                  commit: context.payload.inputs.commit,
                  repo: context.payload.inputs.repo
                }];
              } else {
                throw new Error(`Unsupported event: ${context.eventName}`);
              }
            }

            function get_run_indices() {
              if (context.eventName === 'issue_comment') {
                return [0];
              } else if (context.eventName === 'workflow_dispatch') {
                return Array.from({ length: context.payload.inputs.runs }, (_, i) => i);
              } else {
                throw new Error(`Unsupported event: ${context.eventName}`);
              }
            }

            function get_profile() {
              if (context.eventName === 'issue_comment') {
                return context.payload.comment.body.includes('all') ? 'all' : 'merge';
              } else if (context.eventName === 'workflow_dispatch') {
                return context.payload.inputs.profile;
              } else {
                throw new Error(`Unsupported event: ${context.eventName}`);
              }
            }

            const commits = await get_commits();
            const run_indices = get_run_indices();
            const profile = get_profile();
            const is_nightlys = profile === 'all' ? [false, true] : [profile === 'nightly'];
            const fs = require('fs');

            const runs = [];
            for (const run_index of run_indices) {
              for (const commit of commits) {
                for (const is_nightly of is_nightlys) {
                  const dataCsv = `${process.env.DATA_DIR}/raw/${commit.commit}/${is_nightly ? 'nightly' : 'merge'}-${run_index}.csv`;
                  if (fs.existsSync(dataCsv)) {
                    console.log(`Run ${run_index} for commit ${commit.commit} with profile ${profile} has already been performed.`);
                  } else {
                    console.log(`Scheduling run ${run_index} for commit ${commit.commit} with profile ${profile}.`);
                    runs.push({
                      commit: commit.commit,
                      repo: commit.repo,
                      index: run_index,
                      is_nightly: is_nightly
                    });
                  }
                }
              }
            }

            core.setOutput('runs', JSON.stringify(runs));
            core.setOutput('visualizer_url', `https://dotty-bench.epfl.ch/v3/compare.html#${commits.map(c => c.commit).join(',')}`);
    outputs:
      runs: ${{ steps.generate_runs.outputs.runs }}
      visualizer_url: ${{ steps.generate_runs.outputs.visualizer_url }}

  run:
    name: Run
    needs: ['generate_runs']
    strategy:
      matrix:
        run: ${{fromJson(needs.generate_runs.outputs.runs)}}
      max-parallel: 1
    uses: ./.github/workflows/bench.yml
    with:
      commit: ${{ matrix.run.commit }}
      repo: ${{ matrix.run.repo }}
      run: ${{ matrix.run.index }}
      is_nightly: ${{ matrix.run.is_nightly }}

  finish_comment:
    name: Finish comment
    needs: ['run']
    if: github.event_name == 'issue_comment'
    runs-on: ubuntu-latest
    steps:
      - name: Comment
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: 'Your benchmarks have been run. You can see the results at ${{ needs.generate_runs.outputs.visualizer_url }}.'
            })