bench_matrix.yml

# Run all benchmarks when a comment containing "test performance please"
# comment is posted on a PR.

name: Benchmarks Matrix

on:
  issue_comment:
    types: [created]
  workflow_dispatch:
    inputs:
      commit:
        description: "Commit to benchmark."
        required: true
        type: string
      repo:
        description: "GitHub repository containing the commit to benchmark."
        required: true
        type: string
        default: "mbovel/scala3" # TODO(mbovel): Change to scala/scala3
      runs:
        description: "Number of runs to perform."
        required: true
        type: number
        default: 1
      profile:
        description: "Profile to run: 'merge' to run the benchmarks that are run on every PR merge (shorter), 'nightly' to run nightly benchmarks (longer), or 'all' to run both."
        required: true
        type: choice
        options:
          - merge
          - nightly
          - all

jobs:
  generate_runs:
    name: Generate run definitions
    if: github.event_name == 'workflow_dispatch' || (github.event.issue.pull_request && contains(github.event.comment.body, 'test performance please'))
    runs-on: ubuntu-latest
    steps:
      - id: generate_runs
        uses: actions/github-script@v7
        with:
          script: |
            function get_commits() {
              if (context.eventName === 'issue_comment') {
                const { data } = github.rest.issues.listComments({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: context.issue.number
                });
                const base = {
                  commit: data.base.sha,
                  repo: context.base.repo.full_name
                };
                const head = {
                  commit: data.head.sha,
                  repo: data.head.repo.full_name
                };
                return [base, head];
              } else if (context.eventName === 'workflow_dispatch') {
                return [{
                  commit: context.payload.inputs.commit,
                  repo: context.payload.inputs.repo
                }];
              } else {
                throw new Error(`Unsupported event: ${context .eventName}`);
              }
            }

            function get_runs() {
              if (context.eventName === 'issue_comment') {
                return [0];
              } else if (context.eventName === 'workflow_dispatch') {
                return Array.from({ length: context.payload.inputs.runs }, (_, i) => i);
              } else {
                throw new Error(`Unsupported event: ${context.eventName}`);
              }
            }

            function get_profile() {
              if (context.eventName === 'issue_comment') {
                return context.payload.comment.body.includes('all') ? 'all' : 'merge';
              } else if (context.eventName === 'workflow_dispatch') {
                return context.payload.inputs.profile;
              } else {
                throw new Error(`Unsupported event: ${context.eventName}`);
              }
            }

            const runs = [];
            for (const run of get_runs()) {
              for (const commit of get_commits()) {
                const profile = get_profile();
                const is_nightlys = profile === 'all' ? [false, true] : [profile === 'nightly'];
                for (const is_nightly of is_nightlys) {
                  runs.push({
                    commit: commit.commit,
                    repo: commit.repo,
                    run: run,
                    is_nightly: is_nightly
                  });
                }
              }
            }
            core.setOutput('runs', JSON.stringify(runs));
    outputs:
      runs: ${{ steps.generate_runs.outputs.runs }}

  run:
    name: Run
    needs: ['generate_runs']
    strategy:
      matrix:
        run: ${{fromJson(needs.generate_runs.outputs.runs)}}
    uses: ./.github/workflows/bench.yml
    with:
      commit: ${{ matrix.run.commit }}
      repo: ${{ matrix.run.repo }}
      run: ${{ matrix.run.run }}
      is_nightly: ${{ matrix.run.is_nightly }}