New benchmarks infrastructure #16
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all benchmarks when a comment containing "test performance please" | |
| # comment is posted on a PR. | |
| name: Benchmarks Matrix | |
| on: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| commit: | |
| description: "Commit to benchmark." | |
| required: true | |
| type: string | |
| repo: | |
| description: "GitHub repository containing the commit to benchmark." | |
| required: true | |
| type: string | |
| default: "mbovel/scala3" # TODO(mbovel): Change to scala/scala3 | |
| runs: | |
| description: "Number of runs to perform." | |
| required: true | |
| type: number | |
| default: 1 | |
| profile: | |
| description: "Profile to run: 'merge' to run the benchmarks that are run on every PR merge (shorter), 'nightly' to run nightly benchmarks (longer), or 'all' to run both." | |
| required: true | |
| type: choice | |
| options: | |
| - merge | |
| - nightly | |
| - all | |
| jobs: | |
| generate_runs: | |
| name: Generate run definitions | |
| if: github.event_name == 'workflow_dispatch' || (github.event.issue.pull_request && contains(github.event.comment.body, 'test performance please')) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: generate_runs | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| async function get_commits() { | |
| if (context.eventName === 'issue_comment') { | |
| const {data} = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.payload.issue.number | |
| }); | |
| console.log(data); | |
| const base = { | |
| commit: data.base.sha, | |
| repo: data.base.repo.full_name | |
| }; | |
| const head = { | |
| commit: data.head.sha, | |
| repo: data.head.repo.full_name | |
| }; | |
| return [base, head]; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return [{ | |
| commit: context.payload.inputs.commit, | |
| repo: context.payload.inputs.repo | |
| }]; | |
| } else { | |
| throw new Error(`Unsupported event: ${context .eventName}`); | |
| } | |
| } | |
| function get_runs() { | |
| if (context.eventName === 'issue_comment') { | |
| return [0]; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return Array.from({ length: context.payload.inputs.runs }, (_, i) => i); | |
| } else { | |
| throw new Error(`Unsupported event: ${context.eventName}`); | |
| } | |
| } | |
| function get_profile() { | |
| if (context.eventName === 'issue_comment') { | |
| return context.payload.comment.body.includes('all') ? 'all' : 'merge'; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return context.payload.inputs.profile; | |
| } else { | |
| throw new Error(`Unsupported event: ${context.eventName}`); | |
| } | |
| } | |
| const runs = []; | |
| for (const run of get_runs()) { | |
| for (const commit of await get_commits()) { | |
| const profile = get_profile(); | |
| const is_nightlys = profile === 'all' ? [false, true] : [profile === 'nightly']; | |
| for (const is_nightly of is_nightlys) { | |
| runs.push({ | |
| commit: commit.commit, | |
| repo: commit.repo, | |
| run: run, | |
| is_nightly: is_nightly | |
| }); | |
| } | |
| } | |
| } | |
| core.setOutput('runs', JSON.stringify(runs)); | |
| outputs: | |
| runs: ${{ steps.generate_runs.outputs.runs }} | |
| run: | |
| name: Run | |
| needs: ['generate_runs'] | |
| strategy: | |
| matrix: | |
| run: ${{fromJson(needs.generate_runs.outputs.runs)}} | |
| uses: ./.github/workflows/bench.yml | |
| with: | |
| commit: ${{ matrix.run.commit }} | |
| repo: ${{ matrix.run.repo }} | |
| run: ${{ matrix.run.run }} | |
| is_nightly: ${{ matrix.run.is_nightly }} |