The beginnings of a Django plugin. (#18173)

Currently does two things:
1) Detects Django apps, via their apps.py
2) infers deps from Django migrations to other migrations,
based on the `dependencies` field in the `Migration` classes.

Future changes will add more Django-related functionality.

src/python/pants/backend/experimental/python/framework/django/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/python/framework/django/register.py

@@ -0,0 +1,11 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.framework.django import dependency_inference, detect_apps
+
+
+def rules():
+    return [
+        *detect_apps.rules(),
+        *dependency_inference.rules(),
+    ]

src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py

@@ -13,9 +13,10 @@
 from pants.core.util_rules.source_files import SourceFilesRequest
 from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
 from pants.engine.collection import DeduplicatedCollection
+from pants.engine.environment import EnvironmentName
 from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.process import Process, ProcessResult
-from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.unions import UnionMembership, UnionRule, union
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
@@ -55,7 +56,7 @@ class ParsePythonDependenciesRequest:
     interpreter_constraints: InterpreterConstraints
 
 
-@union(in_scope_types=[])
+@union(in_scope_types=[EnvironmentName])
 class PythonDependencyVisitorRequest:
     pass
 
@@ -75,23 +76,30 @@ class ParserScript:
     env: FrozenDict[str, str]
 
 
-_scripts_module = "pants.backend.python.dependency_inference.scripts"
+_scripts_package = "pants.backend.python.dependency_inference.scripts"
 
 
-@rule_helper
-async def _get_script_digest(relpaths: Iterable[str]) -> Digest:
-    scripts = [read_resource(_scripts_module, relpath) for relpath in relpaths]
+async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest:
+    scripts = [read_resource(scripts_package, filename) for filename in filenames]
     assert all(script is not None for script in scripts)
-    path_prefix = _scripts_module.replace(".", os.path.sep)
-    digest = await Get(
-        Digest,
-        CreateDigest(
-            [
-                FileContent(os.path.join(path_prefix, relpath), script)
-                for relpath, script in zip(relpaths, scripts)
-            ]
-        ),
-    )
+    path_prefix = scripts_package.replace(".", os.path.sep)
+    contents = [
+        FileContent(os.path.join(path_prefix, relpath), script)
+        for relpath, script in zip(filenames, scripts)
+    ]
+
+    # Python 2 requires all the intermediate __init__.py to exist in the sandbox.
+    package = scripts_package
+    while package:
+        contents.append(
+            FileContent(
+                os.path.join(package.replace(".", os.path.sep), "__init__.py"),
+                read_resource(package, "__init__.py"),
+            )
+        )
+        package = package.rpartition(".")[0]
+
+    digest = await Get(Digest, CreateDigest(contents))
     return digest
 
 
@@ -102,29 +110,15 @@ async def get_parser_script(union_membership: UnionMembership) -> ParserScript:
         Get(PythonDependencyVisitor, PythonDependencyVisitorRequest, dvrt())
         for dvrt in dep_visitor_request_types
     )
-    utils = await _get_script_digest(
+    utils = await get_scripts_digest(
+        _scripts_package,
         [
             "dependency_visitor_base.py",
             "main.py",
-        ]
+        ],
     )
 
-    # Python 2 requires all the intermediate __init__.py to exist in the sandbox.
-    init_contents = []
-    module = _scripts_module
-    while module:
-        init_contents.append(
-            FileContent(
-                os.path.join(module.replace(".", os.path.sep), "__init__.py"),
-                read_resource(module, "__init__.py"),
-            )
-        )
-        module = module.rpartition(".")[0]
-    init_scaffolding = await Get(Digest, CreateDigest(init_contents))
-
-    digest = await Get(
-        Digest, MergeDigests([utils, init_scaffolding, *(dv.digest for dv in dep_visitors)])
-    )
+    digest = await Get(Digest, MergeDigests([utils, *(dv.digest for dv in dep_visitors)]))
     env = {
         "VISITOR_CLASSNAMES": "|".join(dv.classname for dv in dep_visitors),
         "PYTHONPATH": ".",
@@ -154,10 +148,10 @@ class GeneralPythonDependencyVisitorRequest(PythonDependencyVisitorRequest):
 @rule
 async def general_parser_script(
     python_infer_subsystem: PythonInferSubsystem,
-    request: GeneralPythonDependencyVisitorRequest,
+    _: GeneralPythonDependencyVisitorRequest,
 ) -> PythonDependencyVisitor:
-    script_digest = await _get_script_digest(["general_dependency_visitor.py"])
-    classname = f"{_scripts_module}.general_dependency_visitor.GeneralDependencyVisitor"
+    script_digest = await get_scripts_digest(_scripts_package, ["general_dependency_visitor.py"])
+    classname = f"{_scripts_package}.general_dependency_visitor.GeneralDependencyVisitor"
     return PythonDependencyVisitor(
         digest=script_digest,
         classname=classname,

src/python/pants/backend/python/dependency_inference/parse_python_dependencies_test.py

@@ -47,15 +47,17 @@ def assert_deps_parsed(
     rule_runner: RuleRunner,
     content: str,
     *,
-    expected_imports: dict[str, ImpInfo] = {},
-    expected_assets: list[str] = [],
+    expected_imports: dict[str, ImpInfo] | None = None,
+    expected_assets: list[str] | None = None,
     filename: str = "project/foo.py",
     constraints: str = ">=3.6",
     string_imports: bool = True,
     string_imports_min_dots: int = 2,
     assets: bool = True,
     assets_min_slashes: int = 1,
 ) -> None:
+    expected_imports = expected_imports or {}
+    expected_assets = expected_assets or []
     rule_runner.set_options(
         [
             f"--python-infer-string-imports={string_imports}",

src/python/pants/backend/python/dependency_inference/scripts/main.py

@@ -3,7 +3,7 @@
 # -*- coding: utf-8 -*-
 
 # NB: This must be compatible with Python 2.7 and 3.5+.
-# NB: If you're needing to debug this, an easy way is to just invoke it on a file.
+# NB: An easy way to debug this is to just invoke it on a file.
 #   E.g.
 #   $ PYTHONPATH=src/python STRING_IMPORTS=y python \
 #     src/python/pants/backend/python/dependency_inference/scripts/main.py FILE

src/python/pants/backend/python/framework/django/BUILD

@@ -0,0 +1,7 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+python_sources(dependencies=["src/python/pants/backend/python/framework/django/scripts"])
+
+python_tests(name="tests")

src/python/pants/backend/python/framework/django/dependency_inference.py

@@ -0,0 +1,49 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+
+from pants.backend.python.dependency_inference.parse_python_dependencies import (
+    PythonDependencyVisitor,
+    PythonDependencyVisitorRequest,
+    get_scripts_digest,
+)
+from pants.backend.python.framework.django.detect_apps import DjangoApps, DjangoAppsRequest
+from pants.engine.fs import CreateDigest, FileContent
+from pants.engine.internals.native_engine import Digest, MergeDigests
+from pants.engine.internals.selectors import Get
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import UnionRule
+from pants.util.frozendict import FrozenDict
+
+
+@dataclass(frozen=True)
+class DjangoDependencyVisitorRequest(PythonDependencyVisitorRequest):
+    pass
+
+
+_scripts_package = "pants.backend.python.framework.django.scripts"
+
+
+@rule
+async def django_parser_script(
+    _: DjangoDependencyVisitorRequest,
+) -> PythonDependencyVisitor:
+    django_apps = await Get(DjangoApps, DjangoAppsRequest())
+    django_apps_digest = await Get(
+        Digest, CreateDigest([FileContent("apps.json", django_apps.to_json())])
+    )
+    scripts_digest = await get_scripts_digest(_scripts_package, ["dependency_visitor.py"])
+    digest = await Get(Digest, MergeDigests([django_apps_digest, scripts_digest]))
+    return PythonDependencyVisitor(
+        digest=digest,
+        classname=f"{_scripts_package}.dependency_visitor.DjangoDependencyVisitor",
+        env=FrozenDict({}),
+    )
+
+
+def rules():
+    return [
+        UnionRule(PythonDependencyVisitorRequest, DjangoDependencyVisitorRequest),
+        *collect_rules(),
+    ]

src/python/pants/backend/python/framework/django/dependency_inference_test.py

@@ -0,0 +1,159 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.python.dependency_inference import parse_python_dependencies
+from pants.backend.python.dependency_inference.parse_python_dependencies import (
+    ParsedPythonDependencies,
+)
+from pants.backend.python.dependency_inference.parse_python_dependencies import (
+    ParsedPythonImportInfo as ImpInfo,
+)
+from pants.backend.python.dependency_inference.parse_python_dependencies import (
+    ParsePythonDependenciesRequest,
+)
+from pants.backend.python.framework.django import dependency_inference, detect_apps
+from pants.backend.python.target_types import PythonSourceField, PythonSourceTarget
+from pants.backend.python.util_rules import pex
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
+from pants.core.util_rules import stripped_source_files
+from pants.engine.addresses import Address
+from pants.testutil.python_interpreter_selection import (
+    skip_unless_python27_present,
+    skip_unless_python37_present,
+    skip_unless_python38_present,
+    skip_unless_python39_present,
+)
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+from pants.util.strutil import softwrap
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *parse_python_dependencies.rules(),
+            *stripped_source_files.rules(),
+            *pex.rules(),
+            *dependency_inference.rules(),
+            *detect_apps.rules(),
+            QueryRule(ParsedPythonDependencies, [ParsePythonDependenciesRequest]),
+        ],
+        target_types=[PythonSourceTarget],
+    )
+
+
+def assert_deps_parsed(
+    rule_runner: RuleRunner,
+    content: str,
+    *,
+    constraints: str,
+    expected_imports: dict[str, ImpInfo] | None = None,
+    expected_assets: list[str] | None = None,
+    filename: str = "path/to/app0/migrations/0001_initial.py",
+) -> None:
+    expected_imports = expected_imports or {}
+    expected_assets = expected_assets or []
+    rule_runner.set_options(
+        [
+            "--no-python-infer-string-imports",
+            "--no-python-infer-assets",
+        ],
+        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+    )
+    rule_runner.write_files(
+        {
+            "BUILD": f"python_source(name='t', source={repr(filename)})",
+            "path/to/app1/BUILD": softwrap(
+                f"""\
+                python_source(
+                  source="apps.py",
+                  interpreter_constraints=['{constraints}'],
+                )
+                """
+            ),
+            "path/to/app1/apps.py": softwrap(
+                """\
+                class App1AppConfig(AppConfig):
+                    name = "path.to.app1"
+                    label = "app1"
+                """
+            ),
+            "another/path/app2/BUILD": softwrap(
+                f"""\
+                python_source(
+                  source="apps.py",
+                  interpreter_constraints=['{constraints}'],
+                )
+                """
+            ),
+            "another/path/app2/apps.py": softwrap(
+                """\
+                class App2AppConfig(AppConfig):
+                    name = "another.path.app2"
+                    label = "app2_label"
+                """
+            ),
+            filename: content,
+        }
+    )
+    tgt = rule_runner.get_target(Address("", target_name="t"))
+    result = rule_runner.request(
+        ParsedPythonDependencies,
+        [
+            ParsePythonDependenciesRequest(
+                tgt[PythonSourceField],
+                InterpreterConstraints([constraints]),
+            )
+        ],
+    )
+    assert dict(result.imports) == expected_imports
+    assert list(result.assets) == sorted(expected_assets)
+
+
+def do_test_migration_dependencies(rule_runner: RuleRunner, constraints: str) -> None:
+    content = dedent(
+        """\
+        class Migration(migrations.Migration):
+            dependencies = [
+              ("app1", "0012_some_migration"),
+              ("app2_label", "0042_another_migration"),
+            ]
+
+            operations = []
+        """
+    )
+    assert_deps_parsed(
+        rule_runner,
+        content,
+        constraints=constraints,
+        expected_imports={
+            "path.to.app1.migrations.0012_some_migration": ImpInfo(lineno=3, weak=True),
+            "another.path.app2.migrations.0042_another_migration": ImpInfo(lineno=4, weak=True),
+        },
+    )
+
+
+@skip_unless_python27_present
+def test_works_with_python2(rule_runner: RuleRunner) -> None:
+    do_test_migration_dependencies(rule_runner, constraints="CPython==2.7.*")
+
+
+@skip_unless_python37_present
+def test_works_with_python37(rule_runner: RuleRunner) -> None:
+    do_test_migration_dependencies(rule_runner, constraints="CPython==3.7.*")
+
+
+@skip_unless_python38_present
+def test_works_with_python38(rule_runner: RuleRunner) -> None:
+    do_test_migration_dependencies(rule_runner, constraints="CPython==3.8.*")
+
+
+@skip_unless_python39_present
+def test_works_with_python39(rule_runner: RuleRunner) -> None:
+    do_test_migration_dependencies(rule_runner, constraints="CPython==3.9.*")