Skip to content

Commit

Permalink
ci: Benchmark CI Step (#3480)
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaReiser authored Mar 16, 2023
1 parent 9ae9cc9 commit aa51ece
Show file tree
Hide file tree
Showing 3 changed files with 206 additions and 15 deletions.
133 changes: 133 additions & 0 deletions .github/workflows/benchmark.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
name: Benchmark

on:
pull_request:
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true

jobs:
run-benchmark:
if: github.event_name == 'pull_request'
name: "Run | ${{ matrix.os }}"
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}

steps:
- name: "PR - Checkout Branch"
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: "PR - Install Rust toolchain"
run: rustup show

- uses: Swatinem/rust-cache@v1

- name: "PR - Build benchmarks"
uses: actions-rs/cargo@v1
with:
command: bench
args: -p ruff_benchmark --no-run

- name: "PR - Run benchmarks"
run: cargo benchmark --save-baseline=pr

- name: "Main - Checkout Branch"
uses: actions/checkout@v3
with:
clean: false
ref: main

- name: "Main - Install Rust toolchain"
run: rustup show

- name: "Main - Build benchmarks"
uses: actions-rs/cargo@v1
with:
command: bench
args: -p ruff_benchmark --no-run

- name: "Main - Run benchmarks"
run: cargo benchmark --save-baseline=main

- name: "Upload benchmark results"
uses: actions/upload-artifact@v3
with:
name: benchmark-results-${{ matrix.os }}
path: ./target/criterion

# Cleanup
- name: Remove Criterion Artifact
uses: JesseTG/[email protected]
with:
path: ./target/criterion

benchmark-compare:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
name: Compare
needs:
- run-benchmark

steps:
- name: "Install Rust toolchain"
run: rustup show

- name: "Install critcmp"
# Use debug build: Building takes much longer than the "slowness" of using the debug build.
run: cargo install --debug critcmp

- name: "Linux | Download PR benchmark results"
uses: actions/download-artifact@v3
with:
name: benchmark-results-ubuntu-latest
path: ./target/criterion

- name: "Linux | Compare benchmark results"
shell: bash
run: |
echo "### Benchmark" >> summary.md
echo "#### Linux" >> summary.md
echo "\`\`\`" >> summary.md
critcmp main pr >> summary.md
echo "\`\`\`" >> summary.md
echo "" >> summary.md
- name: "Linux | Cleanup benchmark results"
run: rm -rf ./target/criterion

- name: "Windows | Download PR benchmark results"
uses: actions/download-artifact@v3
with:
name: benchmark-results-windows-latest
path: ./target/criterion

- name: "Windows | Compare benchmark results"
shell: bash
run: |
echo "#### Windows" >> summary.md
echo "\`\`\`" >> summary.md
critcmp main pr >> summary.md
echo "\`\`\`" >> summary.md
echo "" >> summary.md
echo ${{ github.event.pull_request.number }} > pr-number
cat summary.md > $GITHUB_STEP_SUMMARY
- uses: actions/upload-artifact@v3
name: Upload PR Number
with:
name: pr-number
path: pr-number

- uses: actions/upload-artifact@v3
name: Upload Summary
with:
name: summary
path: summary.md
19 changes: 16 additions & 3 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ jobs:
# Setting RUSTDOCFLAGS because `cargo doc --check` isn't yet implemented (https://github.com/rust-lang/cargo/issues/10025).
RUSTDOCFLAGS: "-D warnings"
- uses: actions/upload-artifact@v3
if: ${{ matrix.os == 'ubuntu-latest' }}
with:
name: ruff
path: target/debug/ruff
Expand Down Expand Up @@ -139,27 +140,39 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: "3.11"

- uses: actions/download-artifact@v3
name: Download Ruff binary
id: ruff-target
with:
name: ruff
path: target/debug

- uses: dawidd6/action-download-artifact@v2
name: Download base results
with:
name: ruff
branch: ${{ github.event.pull_request.base.ref }}
check_artifacts: true

- name: Run ecosystem check
run: |
# Make executable, since artifact download doesn't preserve this
chmod +x ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
scripts/check_ecosystem.py ruff ${{ steps.ruff-target.outputs.download-path }}/ruff | tee ecosystem-result
cat ecosystem-result > $GITHUB_STEP_SUMMARY
echo ${{ github.event.number }} > pr-number
- uses: actions/upload-artifact@v3
name: Upload PR Number
with:
name: pr-number
path: pr-number

- uses: actions/upload-artifact@v3
name: Upload Results
with:
name: ecosystem-result
path: |
ecosystem-result
pr-number
path: ecosystem-result
69 changes: 57 additions & 12 deletions .github/workflows/pr-comment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: PR Check Comment

on:
workflow_run:
workflows: [CI]
workflows: [CI, Benchmark]
types: [completed]
workflow_dispatch:
inputs:
Expand All @@ -17,22 +17,67 @@ jobs:
comment:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dawidd6/action-download-artifact@v2
id: download-result
name: Download PR Number
with:
name: pr-number
run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }}
if_no_artifact_found: ignore

- name: Extract PR Number
id: pr-number
run: |
if [[ -f pr-number ]]
then
echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
fi
- uses: dawidd6/action-download-artifact@v2
name: "Download Ecosystem Result"
id: download-ecosystem-result
if: steps.pr-number.outputs.pr-number
with:
name: ecosystem-result
workflow: ci.yaml
run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }}
pr: ${{ steps.pr-number.outputs.pr-number }}
path: pr/ecosystem
if_no_artifact_found: ignore
- if: steps.download-result.outputs.found_artifact
id: result

- uses: dawidd6/action-download-artifact@v2
name: "Download Benchmark Result"
id: download-benchmark-result
if: steps.pr-number.outputs.pr-number
with:
name: summary
workflow: benchmark.yaml
pr: ${{ steps.pr-number.outputs.pr-number }}
path: pr/benchmark
if_no_artifact_found: ignore

- name: Generate Comment
id: generate-comment
if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
run: |
echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
- name: Create comment
if: steps.download-result.outputs.found_artifact
echo 'comment<<EOF' >> $GITHUB_OUTPUT
echo '## PR Check Results' >> $GITHUB_OUTPUT
if [[ -f pr/ecosystem/ecosystem-result ]]
then
echo "### Ecosystem" >> $GITHUB_OUTPUT
cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
fi
if [[ -f pr/benchmark/summary.md ]]
then
cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
fi
echo 'EOF' >> $GITHUB_OUTPUT
- name: Create or update comment
if: steps.generate-comment.outputs.comment
uses: thollander/actions-comment-pull-request@v2
with:
pr_number: ${{ steps.result.outputs.pr-number }}
filePath: ecosystem-result
comment_tag: ecosystem-results
pr_number: ${{ steps.pr-number.outputs.pr-number }}
message: ${{ steps.generate-comment.outputs.comment }}
comment_tag: PR Check Results

0 comments on commit aa51ece

Please sign in to comment.