Rewrite ecosystem checks and add ruff format reports

.github/workflows/ci.yaml

@@ -43,6 +43,7 @@ jobs:
               - "!crates/ruff_dev/**"
               - "!crates/ruff_shrinking/**"
               - scripts/*
+              - .github/workflows/ci.yaml
 
             formatter:
               - Cargo.toml
@@ -57,6 +58,7 @@ jobs:
               - crates/ruff_python_parser/**
               - crates/ruff_dev/**
               - scripts/*
+              - .github/workflows/ci.yaml
 
   cargo-fmt:
     name: "cargo fmt"
@@ -182,35 +184,56 @@ jobs:
       - cargo-test-linux
       - determine_changes
     # Only runs on pull requests, since that is the only we way we can find the base version for comparison.
-    if: github.event_name == 'pull_request' && needs.determine_changes.outputs.linter == 'true'
+    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - uses: actions/download-artifact@v3
-        name: Download Ruff binary
+        name: Download comparison Ruff binary
         id: ruff-target
         with:
           name: ruff
           path: target/debug
 
       - uses: dawidd6/action-download-artifact@v2
-        name: Download base results
+        name: Download baseline Ruff binary
         with:
           name: ruff
           branch: ${{ github.event.pull_request.base.ref }}
           check_artifacts: true
 
-      - name: Run ecosystem check
+      - name: Install ruff-ecosystem
+        run: |
+          pip install ./python/ruff-ecosystem
+
+      - name: Run `ruff check` ecosystem check
+        if: ${{ needs.determine_changes.outputs.linter == 'true' }}
         run: |
           # Make executable, since artifact download doesn't preserve this
-          chmod +x ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
+          chmod +x ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
+
+          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-check
 
-          scripts/check_ecosystem.py ruff ${{ steps.ruff-target.outputs.download-path }}/ruff | tee ecosystem-result
-          cat ecosystem-result > $GITHUB_STEP_SUMMARY
+          cat ecosystem-result-check > $GITHUB_STEP_SUMMARY
+          cat ecosystem-result-check > ecosystem-result
+          echo "" >> ecosystem-result
 
+      - name: Run `ruff format` ecosystem check
+        if: ${{ needs.determine_changes.outputs.formatter == 'true' }}
+        run: |
+          # Make executable, since artifact download doesn't preserve this
+          chmod +x ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
+
+          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-format
+
+          cat ecosystem-result-format > $GITHUB_STEP_SUMMARY
+          cat ecosystem-result-format >> ecosystem-result
+
+      - name: Export pull request number
+        run: |
           echo ${{ github.event.number }} > pr-number
 
       - uses: actions/upload-artifact@v3

python/ruff-ecosystem/README.md

@@ -0,0 +1,46 @@
+# ruff-ecosystem
+
+Compare lint and format results for two different ruff versions (e.g. main and a PR) on real world projects.
+
+## Installation
+
+From the Ruff project root, install with `pip`:
+
+```shell
+pip install -e ./python/ruff-ecosystem
+```
+
+## Usage
+
+```shell
+ruff-ecosystem <check | format> <baseline executable> <comparison executable>
+```
+
+Note executable paths may be absolute, relative to the current working directory, or will be looked up in the
+current Python environment and PATH.
+
+Run `ruff check` ecosystem checks comparing your debug build to your system Ruff:
+
+```shell
+ruff-ecosystem check ruff "./target/debug/ruff"
+```
+
+Run `ruff format` ecosystem checks comparing your debug build to your system Ruff:
+
+```shell
+ruff-ecosystem format ruff "./target/debug/ruff"
+```
+
+## Development
+
+When developing, it can be useful to set the `--pdb` flag to drop into a debugger on failure:
+
+```shell
+ruff-ecosystem check ruff "./target/debug/ruff" --pdb
+```
+
+You can also provide a path to cache checkouts to speed up repeated runs:
+
+```shell
+ruff-ecosystem check ruff "./target/debug/ruff" --cache ./repos
+```

python/ruff-ecosystem/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ruff-ecosystem"
+version = "0.0.0"
+dependencies = ["unidiff==0.7.5"]
+
+[project.scripts]
+ruff-ecosystem = "ruff_ecosystem.cli:entrypoint"
+
+[tool.ruff]
+extend-select = ["I"]
+preview = true

python/ruff-ecosystem/ruff_ecosystem/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("ruff-ecosystem")

python/ruff-ecosystem/ruff_ecosystem/__main__.py

@@ -0,0 +1,8 @@
+"""
+Enables usage with `python -m ruff_ecosystem`
+"""
+
+import ruff_ecosystem.cli
+
+if __name__ == "__main__":
+    ruff_ecosystem.cli.entrypoint()