New benchmarks infrastructure #20
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all benchmarks when a comment containing "test performance please" | |
| # comment is posted on a PR. | |
| name: Benchmarks Matrix | |
| on: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| commit: | |
| description: "Commit to benchmark." | |
| required: true | |
| type: string | |
| repo: | |
| description: "GitHub repository containing the commit to benchmark." | |
| required: true | |
| type: string | |
| default: "mbovel/scala3" # TODO(mbovel): Change to scala/scala3 | |
| runs: | |
| description: "Number of runs to perform." | |
| required: true | |
| type: number | |
| default: 1 | |
| profile: | |
| description: "Profile to run: 'merge' to run the benchmarks that are run on every PR merge (shorter), 'nightly' to run nightly benchmarks (longer), or 'all' to run both." | |
| required: true | |
| type: choice | |
| options: | |
| - merge | |
| - nightly | |
| - all | |
| jobs: | |
| start_comment: | |
| name: Start comment | |
| if: github.event_name == 'issue_comment' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `Thanks for your request. Your benchmarks will be run shortly.\nYou can follow the progress at https://github.com/mbovel/dotty/actions/runs/${context.runId}.` | |
| }) | |
| generate_runs: | |
| name: Generate run definitions | |
| if: github.event_name == 'workflow_dispatch' || (github.event.issue.pull_request && contains(github.event.comment.body, 'test performance please')) | |
| runs-on: ['self-hosted', 'benchmarks'] | |
| env: | |
| # Path to the directory where the benchmark data is stored on the runner. | |
| # Keep in sync with the value in .github/workflows/bench.yml. | |
| DATA_DIR: /home/scalabenchs/bench-data-v3 | |
| steps: | |
| - id: generate_runs | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| async function get_commits() { | |
| if (context.eventName === 'issue_comment') { | |
| const {data} = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.payload.issue.number | |
| }); | |
| const base = { | |
| commit: data.base.sha, | |
| repo: data.base.repo.full_name | |
| }; | |
| core.setOutput('base', base.commit); | |
| const head = { | |
| commit: data.head.sha, | |
| repo: data.head.repo.full_name | |
| }; | |
| core.setOutput('head', head.commit); | |
| return [base, head]; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return [{ | |
| commit: context.payload.inputs.commit, | |
| repo: context.payload.inputs.repo | |
| }]; | |
| } else { | |
| throw new Error(`Unsupported event: ${context.eventName}`); | |
| } | |
| } | |
| function get_run_indices() { | |
| if (context.eventName === 'issue_comment') { | |
| return [0]; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return Array.from({ length: context.payload.inputs.runs }, (_, i) => i); | |
| } else { | |
| throw new Error(`Unsupported event: ${context.eventName}`); | |
| } | |
| } | |
| function get_profile() { | |
| if (context.eventName === 'issue_comment') { | |
| return context.payload.comment.body.includes('all') ? 'all' : 'merge'; | |
| } else if (context.eventName === 'workflow_dispatch') { | |
| return context.payload.inputs.profile; | |
| } else { | |
| throw new Error(`Unsupported event: ${context.eventName}`); | |
| } | |
| } | |
| const commits = await get_commits(); | |
| const run_indices = get_run_indices(); | |
| const profile = get_profile(); | |
| const is_nightlys = profile === 'all' ? [false, true] : [profile === 'nightly']; | |
| const fs = require('fs'); | |
| const runs = []; | |
| for (const run_index of run_indices) { | |
| for (const commit of commits) { | |
| for (const is_nightly of is_nightlys) { | |
| const dataCsv = `${process.env.DATA_DIR}/raw/${commit.commit}/${is_nightly ? 'nightly' : 'merge'}-${run_index}.csv`; | |
| if (fs.existsSync(dataCsv)) { | |
| console.log(`Run ${run_index} for commit ${commit.commit} with profile ${profile} has already been performed.`); | |
| } else { | |
| console.log(`Scheduling run ${run_index} for commit ${commit.commit} with profile ${profile}.`); | |
| runs.push({ | |
| commit: commit.commit, | |
| repo: commit.repo, | |
| index: run_index, | |
| is_nightly: is_nightly | |
| }); | |
| } | |
| } | |
| } | |
| } | |
| core.setOutput('runs', JSON.stringify(runs)); | |
| core.setOutput('visualizer_url', `https://dotty-bench.epfl.ch/v3/compare.html#${commits.map(c => c.commit).join(',')}`); | |
| outputs: | |
| runs: ${{ steps.generate_runs.outputs.runs }} | |
| visualizer_url: ${{ steps.generate_runs.outputs.visualizer_url }} | |
| run: | |
| name: Run | |
| needs: ['generate_runs'] | |
| strategy: | |
| matrix: | |
| run: ${{fromJson(needs.generate_runs.outputs.runs)}} | |
| max-parallel: 1 | |
| uses: ./.github/workflows/bench.yml | |
| with: | |
| commit: ${{ matrix.run.commit }} | |
| repo: ${{ matrix.run.repo }} | |
| run: ${{ matrix.run.index }} | |
| is_nightly: ${{ matrix.run.is_nightly }} | |
| finish_comment: | |
| name: Finish comment | |
| needs: ['run'] | |
| if: github.event_name == 'issue_comment' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: 'Your benchmarks have been run. You can see the results at ${{ needs.generate_runs.outputs.visualizer_url }}.' | |
| }) |