Allow config-file overrides in ecosystem checks (#9286)

Adds the ability to override `ruff.toml` or `pyproject.toml` settings
per-project during ecosystem checks.

Exploring this as a fix for the `setuptools` project error. 

Also useful for including Jupyter Notebooks in the ecosystem checks, see
#9293

Note the remaining `sphinx` project error is resolved in #9294

python/ruff-ecosystem/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "ruff-ecosystem"
 version = "0.0.0"
-dependencies = ["unidiff==0.7.5"]
+dependencies = ["unidiff==0.7.5", "tomli_w==1.0.0", "tomli==2.0.1"]
 
 [project.scripts]
 ruff-ecosystem = "ruff_ecosystem.cli:entrypoint"

python/ruff-ecosystem/ruff_ecosystem/check.py

@@ -28,7 +28,12 @@
 )
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import CheckOptions, ClonedRepository, Project
+    from ruff_ecosystem.projects import (
+        CheckOptions,
+        ClonedRepository,
+        ConfigOverrides,
+        Project,
+    )
 
 
 # Matches lines that are summaries rather than diagnostics
@@ -477,25 +482,27 @@ async def compare_check(
     ruff_baseline_executable: Path,
     ruff_comparison_executable: Path,
     options: CheckOptions,
+    config_overrides: ConfigOverrides,
     cloned_repo: ClonedRepository,
 ) -> Comparison:
-    async with asyncio.TaskGroup() as tg:
-        baseline_task = tg.create_task(
-            ruff_check(
-                executable=ruff_baseline_executable.resolve(),
-                path=cloned_repo.path,
-                name=cloned_repo.fullname,
-                options=options,
-            ),
-        )
-        comparison_task = tg.create_task(
-            ruff_check(
-                executable=ruff_comparison_executable.resolve(),
-                path=cloned_repo.path,
-                name=cloned_repo.fullname,
-                options=options,
-            ),
-        )
+    with config_overrides.patch_config(cloned_repo.path, options.preview):
+        async with asyncio.TaskGroup() as tg:
+            baseline_task = tg.create_task(
+                ruff_check(
+                    executable=ruff_baseline_executable.resolve(),
+                    path=cloned_repo.path,
+                    name=cloned_repo.fullname,
+                    options=options,
+                ),
+            )
+            comparison_task = tg.create_task(
+                ruff_check(
+                    executable=ruff_comparison_executable.resolve(),
+                    path=cloned_repo.path,
+                    name=cloned_repo.fullname,
+                    options=options,
+                ),
+            )
 
     baseline_output, comparison_output = (
         baseline_task.result(),

python/ruff-ecosystem/ruff_ecosystem/defaults.py

@@ -1,7 +1,13 @@
 """
 Default projects for ecosystem checks
 """
-from ruff_ecosystem.projects import CheckOptions, FormatOptions, Project, Repository
+from ruff_ecosystem.projects import (
+    CheckOptions,
+    ConfigOverrides,
+    FormatOptions,
+    Project,
+    Repository,
+)
 
 # TODO(zanieb): Consider exporting this as JSON and loading from there instead
 DEFAULT_TARGETS = [
@@ -45,7 +51,14 @@
     Project(repo=Repository(owner="pypa", name="build", ref="main")),
     Project(repo=Repository(owner="pypa", name="cibuildwheel", ref="main")),
     Project(repo=Repository(owner="pypa", name="pip", ref="main")),
-    Project(repo=Repository(owner="pypa", name="setuptools", ref="main")),
+    Project(
+        repo=Repository(owner="pypa", name="setuptools", ref="main"),
+        # Since `setuptools` opts into the "preserve" quote style which
+        # require preview mode, we must disable it during the `--no-preview` run
+        config_overrides=ConfigOverrides(
+            when_no_preview={"format.quote-style": "double"}
+        ),
+    ),
     Project(repo=Repository(owner="python", name="mypy", ref="master")),
     Project(
         repo=Repository(

python/ruff-ecosystem/ruff_ecosystem/format.py

@@ -18,7 +18,7 @@
 from ruff_ecosystem.types import Comparison, Diff, Result, ToolError
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import ClonedRepository, FormatOptions
+    from ruff_ecosystem.projects import ClonedRepository, ConfigOverrides, FormatOptions
 
 
 def markdown_format_result(result: Result) -> str:
@@ -137,10 +137,17 @@ async def compare_format(
     ruff_baseline_executable: Path,
     ruff_comparison_executable: Path,
     options: FormatOptions,
+    config_overrides: ConfigOverrides,
     cloned_repo: ClonedRepository,
     format_comparison: FormatComparison,
 ):
-    args = (ruff_baseline_executable, ruff_comparison_executable, options, cloned_repo)
+    args = (
+        ruff_baseline_executable,
+        ruff_comparison_executable,
+        options,
+        config_overrides,
+        cloned_repo,
+    )
     match format_comparison:
         case FormatComparison.ruff_then_ruff:
             coro = format_then_format(Formatter.ruff, *args)
@@ -162,25 +169,27 @@ async def format_then_format(
     ruff_baseline_executable: Path,
     ruff_comparison_executable: Path,
     options: FormatOptions,
+    config_overrides: ConfigOverrides,
     cloned_repo: ClonedRepository,
 ) -> Sequence[str]:
-    # Run format to get the baseline
-    await format(
-        formatter=baseline_formatter,
-        executable=ruff_baseline_executable.resolve(),
-        path=cloned_repo.path,
-        name=cloned_repo.fullname,
-        options=options,
-    )
-    # Then get the diff from stdout
-    diff = await format(
-        formatter=Formatter.ruff,
-        executable=ruff_comparison_executable.resolve(),
-        path=cloned_repo.path,
-        name=cloned_repo.fullname,
-        options=options,
-        diff=True,
-    )
+    with config_overrides.patch_config(cloned_repo.path, options.preview):
+        # Run format to get the baseline
+        await format(
+            formatter=baseline_formatter,
+            executable=ruff_baseline_executable.resolve(),
+            path=cloned_repo.path,
+            name=cloned_repo.fullname,
+            options=options,
+        )
+        # Then get the diff from stdout
+        diff = await format(
+            formatter=Formatter.ruff,
+            executable=ruff_comparison_executable.resolve(),
+            path=cloned_repo.path,
+            name=cloned_repo.fullname,
+            options=options,
+            diff=True,
+        )
     return diff
 
 
@@ -189,32 +198,39 @@ async def format_and_format(
     ruff_baseline_executable: Path,
     ruff_comparison_executable: Path,
     options: FormatOptions,
+    config_overrides: ConfigOverrides,
     cloned_repo: ClonedRepository,
 ) -> Sequence[str]:
-    # Run format without diff to get the baseline
-    await format(
-        formatter=baseline_formatter,
-        executable=ruff_baseline_executable.resolve(),
-        path=cloned_repo.path,
-        name=cloned_repo.fullname,
-        options=options,
-    )
+    with config_overrides.patch_config(cloned_repo.path, options.preview):
+        # Run format without diff to get the baseline
+        await format(
+            formatter=baseline_formatter,
+            executable=ruff_baseline_executable.resolve(),
+            path=cloned_repo.path,
+            name=cloned_repo.fullname,
+            options=options,
+        )
+
     # Commit the changes
     commit = await cloned_repo.commit(
         message=f"Formatted with baseline {ruff_baseline_executable}"
     )
     # Then reset
     await cloned_repo.reset()
-    # Then run format again
-    await format(
-        formatter=Formatter.ruff,
-        executable=ruff_comparison_executable.resolve(),
-        path=cloned_repo.path,
-        name=cloned_repo.fullname,
-        options=options,
-    )
+
+    with config_overrides.patch_config(cloned_repo.path, options.preview):
+        # Then run format again
+        await format(
+            formatter=Formatter.ruff,
+            executable=ruff_comparison_executable.resolve(),
+            path=cloned_repo.path,
+            name=cloned_repo.fullname,
+            options=options,
+        )
+
     # Then get the diff from the commit
     diff = await cloned_repo.diff(commit)
+
     return diff
 
 

python/ruff-ecosystem/ruff_ecosystem/main.py

@@ -113,11 +113,17 @@ async def clone_and_compare(
 
     match command:
         case RuffCommand.check:
-            compare, options, kwargs = (compare_check, target.check_options, {})
+            compare, options, overrides, kwargs = (
+                compare_check,
+                target.check_options,
+                target.config_overrides,
+                {},
+            )
         case RuffCommand.format:
-            compare, options, kwargs = (
+            compare, options, overrides, kwargs = (
                 compare_format,
                 target.format_options,
+                target.config_overrides,
                 {"format_comparison": format_comparison},
             )
         case _:
@@ -131,6 +137,7 @@ async def clone_and_compare(
             baseline_executable,
             comparison_executable,
             options,
+            overrides,
             cloned_repo,
             **kwargs,
         )