From c2d651b65ede46466ac33d6d475ef8f87ba26720 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 9 Feb 2023 12:53:50 -0500 Subject: [PATCH] The beginnings of a Django plugin. (#18173) Currently does two things: 1) Detects Django apps, via their apps.py 2) infers deps from Django migrations to other migrations, based on the `dependencies` field in the `Migration` classes. Future changes will add more Django-related functionality. --- .../python/framework/django/BUILD | 4 + .../python/framework/django/__init__.py | 0 .../python/framework/django/register.py | 11 ++ .../parse_python_dependencies.py | 68 ++++---- .../parse_python_dependencies_test.py | 6 +- .../dependency_inference/scripts/main.py | 2 +- .../backend/python/framework/django/BUILD | 7 + .../python/framework/django/__init__.py | 0 .../framework/django/dependency_inference.py | 49 ++++++ .../django/dependency_inference_test.py | 159 ++++++++++++++++++ .../python/framework/django/detect_apps.py | 128 ++++++++++++++ .../framework/django/detect_apps_test.py | 144 ++++++++++++++++ .../python/framework/django/scripts/BUILD | 4 + .../framework/django/scripts/__init__.py | 0 .../framework/django/scripts/app_detector.py | 99 +++++++++++ .../django/scripts/dependency_visitor.py | 55 ++++++ .../util_rules/interpreter_constraints.py | 7 - 17 files changed, 696 insertions(+), 47 deletions(-) create mode 100644 src/python/pants/backend/experimental/python/framework/django/BUILD create mode 100644 src/python/pants/backend/experimental/python/framework/django/__init__.py create mode 100644 src/python/pants/backend/experimental/python/framework/django/register.py create mode 100644 src/python/pants/backend/python/framework/django/BUILD create mode 100644 src/python/pants/backend/python/framework/django/__init__.py create mode 100644 src/python/pants/backend/python/framework/django/dependency_inference.py create mode 100644 src/python/pants/backend/python/framework/django/dependency_inference_test.py create mode 100644 src/python/pants/backend/python/framework/django/detect_apps.py create mode 100644 src/python/pants/backend/python/framework/django/detect_apps_test.py create mode 100644 src/python/pants/backend/python/framework/django/scripts/BUILD create mode 100644 src/python/pants/backend/python/framework/django/scripts/__init__.py create mode 100644 src/python/pants/backend/python/framework/django/scripts/app_detector.py create mode 100644 src/python/pants/backend/python/framework/django/scripts/dependency_visitor.py diff --git a/src/python/pants/backend/experimental/python/framework/django/BUILD b/src/python/pants/backend/experimental/python/framework/django/BUILD new file mode 100644 index 00000000000..68aaac88424 --- /dev/null +++ b/src/python/pants/backend/experimental/python/framework/django/BUILD @@ -0,0 +1,4 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/src/python/pants/backend/experimental/python/framework/django/__init__.py b/src/python/pants/backend/experimental/python/framework/django/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/experimental/python/framework/django/register.py b/src/python/pants/backend/experimental/python/framework/django/register.py new file mode 100644 index 00000000000..2ea609849da --- /dev/null +++ b/src/python/pants/backend/experimental/python/framework/django/register.py @@ -0,0 +1,11 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.backend.python.framework.django import dependency_inference, detect_apps + + +def rules(): + return [ + *detect_apps.rules(), + *dependency_inference.rules(), + ] diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index 04dfdebffbd..3d102cee61c 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -13,9 +13,10 @@ from pants.core.util_rules.source_files import SourceFilesRequest from pants.core.util_rules.stripped_source_files import StrippedSourceFiles from pants.engine.collection import DeduplicatedCollection +from pants.engine.environment import EnvironmentName from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.process import Process, ProcessResult -from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.unions import UnionMembership, UnionRule, union from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel @@ -55,7 +56,7 @@ class ParsePythonDependenciesRequest: interpreter_constraints: InterpreterConstraints -@union(in_scope_types=[]) +@union(in_scope_types=[EnvironmentName]) class PythonDependencyVisitorRequest: pass @@ -75,23 +76,30 @@ class ParserScript: env: FrozenDict[str, str] -_scripts_module = "pants.backend.python.dependency_inference.scripts" +_scripts_package = "pants.backend.python.dependency_inference.scripts" -@rule_helper -async def _get_script_digest(relpaths: Iterable[str]) -> Digest: - scripts = [read_resource(_scripts_module, relpath) for relpath in relpaths] +async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest: + scripts = [read_resource(scripts_package, filename) for filename in filenames] assert all(script is not None for script in scripts) - path_prefix = _scripts_module.replace(".", os.path.sep) - digest = await Get( - Digest, - CreateDigest( - [ - FileContent(os.path.join(path_prefix, relpath), script) - for relpath, script in zip(relpaths, scripts) - ] - ), - ) + path_prefix = scripts_package.replace(".", os.path.sep) + contents = [ + FileContent(os.path.join(path_prefix, relpath), script) + for relpath, script in zip(filenames, scripts) + ] + + # Python 2 requires all the intermediate __init__.py to exist in the sandbox. + package = scripts_package + while package: + contents.append( + FileContent( + os.path.join(package.replace(".", os.path.sep), "__init__.py"), + read_resource(package, "__init__.py"), + ) + ) + package = package.rpartition(".")[0] + + digest = await Get(Digest, CreateDigest(contents)) return digest @@ -102,29 +110,15 @@ async def get_parser_script(union_membership: UnionMembership) -> ParserScript: Get(PythonDependencyVisitor, PythonDependencyVisitorRequest, dvrt()) for dvrt in dep_visitor_request_types ) - utils = await _get_script_digest( + utils = await get_scripts_digest( + _scripts_package, [ "dependency_visitor_base.py", "main.py", - ] + ], ) - # Python 2 requires all the intermediate __init__.py to exist in the sandbox. - init_contents = [] - module = _scripts_module - while module: - init_contents.append( - FileContent( - os.path.join(module.replace(".", os.path.sep), "__init__.py"), - read_resource(module, "__init__.py"), - ) - ) - module = module.rpartition(".")[0] - init_scaffolding = await Get(Digest, CreateDigest(init_contents)) - - digest = await Get( - Digest, MergeDigests([utils, init_scaffolding, *(dv.digest for dv in dep_visitors)]) - ) + digest = await Get(Digest, MergeDigests([utils, *(dv.digest for dv in dep_visitors)])) env = { "VISITOR_CLASSNAMES": "|".join(dv.classname for dv in dep_visitors), "PYTHONPATH": ".", @@ -154,10 +148,10 @@ class GeneralPythonDependencyVisitorRequest(PythonDependencyVisitorRequest): @rule async def general_parser_script( python_infer_subsystem: PythonInferSubsystem, - request: GeneralPythonDependencyVisitorRequest, + _: GeneralPythonDependencyVisitorRequest, ) -> PythonDependencyVisitor: - script_digest = await _get_script_digest(["general_dependency_visitor.py"]) - classname = f"{_scripts_module}.general_dependency_visitor.GeneralDependencyVisitor" + script_digest = await get_scripts_digest(_scripts_package, ["general_dependency_visitor.py"]) + classname = f"{_scripts_package}.general_dependency_visitor.GeneralDependencyVisitor" return PythonDependencyVisitor( digest=script_digest, classname=classname, diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies_test.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies_test.py index cf6e3960726..fc97b68f3a8 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies_test.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies_test.py @@ -47,8 +47,8 @@ def assert_deps_parsed( rule_runner: RuleRunner, content: str, *, - expected_imports: dict[str, ImpInfo] = {}, - expected_assets: list[str] = [], + expected_imports: dict[str, ImpInfo] | None = None, + expected_assets: list[str] | None = None, filename: str = "project/foo.py", constraints: str = ">=3.6", string_imports: bool = True, @@ -56,6 +56,8 @@ def assert_deps_parsed( assets: bool = True, assets_min_slashes: int = 1, ) -> None: + expected_imports = expected_imports or {} + expected_assets = expected_assets or [] rule_runner.set_options( [ f"--python-infer-string-imports={string_imports}", diff --git a/src/python/pants/backend/python/dependency_inference/scripts/main.py b/src/python/pants/backend/python/dependency_inference/scripts/main.py index a4ceef3a649..350aada03ee 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/main.py +++ b/src/python/pants/backend/python/dependency_inference/scripts/main.py @@ -3,7 +3,7 @@ # -*- coding: utf-8 -*- # NB: This must be compatible with Python 2.7 and 3.5+. -# NB: If you're needing to debug this, an easy way is to just invoke it on a file. +# NB: An easy way to debug this is to just invoke it on a file. # E.g. # $ PYTHONPATH=src/python STRING_IMPORTS=y python \ # src/python/pants/backend/python/dependency_inference/scripts/main.py FILE diff --git a/src/python/pants/backend/python/framework/django/BUILD b/src/python/pants/backend/python/framework/django/BUILD new file mode 100644 index 00000000000..e93e95d3ac2 --- /dev/null +++ b/src/python/pants/backend/python/framework/django/BUILD @@ -0,0 +1,7 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +python_sources(dependencies=["src/python/pants/backend/python/framework/django/scripts"]) + +python_tests(name="tests") diff --git a/src/python/pants/backend/python/framework/django/__init__.py b/src/python/pants/backend/python/framework/django/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/framework/django/dependency_inference.py b/src/python/pants/backend/python/framework/django/dependency_inference.py new file mode 100644 index 00000000000..5f5dcab2bcc --- /dev/null +++ b/src/python/pants/backend/python/framework/django/dependency_inference.py @@ -0,0 +1,49 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass + +from pants.backend.python.dependency_inference.parse_python_dependencies import ( + PythonDependencyVisitor, + PythonDependencyVisitorRequest, + get_scripts_digest, +) +from pants.backend.python.framework.django.detect_apps import DjangoApps, DjangoAppsRequest +from pants.engine.fs import CreateDigest, FileContent +from pants.engine.internals.native_engine import Digest, MergeDigests +from pants.engine.internals.selectors import Get +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.frozendict import FrozenDict + + +@dataclass(frozen=True) +class DjangoDependencyVisitorRequest(PythonDependencyVisitorRequest): + pass + + +_scripts_package = "pants.backend.python.framework.django.scripts" + + +@rule +async def django_parser_script( + _: DjangoDependencyVisitorRequest, +) -> PythonDependencyVisitor: + django_apps = await Get(DjangoApps, DjangoAppsRequest()) + django_apps_digest = await Get( + Digest, CreateDigest([FileContent("apps.json", django_apps.to_json())]) + ) + scripts_digest = await get_scripts_digest(_scripts_package, ["dependency_visitor.py"]) + digest = await Get(Digest, MergeDigests([django_apps_digest, scripts_digest])) + return PythonDependencyVisitor( + digest=digest, + classname=f"{_scripts_package}.dependency_visitor.DjangoDependencyVisitor", + env=FrozenDict({}), + ) + + +def rules(): + return [ + UnionRule(PythonDependencyVisitorRequest, DjangoDependencyVisitorRequest), + *collect_rules(), + ] diff --git a/src/python/pants/backend/python/framework/django/dependency_inference_test.py b/src/python/pants/backend/python/framework/django/dependency_inference_test.py new file mode 100644 index 00000000000..c35b1b8db1d --- /dev/null +++ b/src/python/pants/backend/python/framework/django/dependency_inference_test.py @@ -0,0 +1,159 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.python.dependency_inference import parse_python_dependencies +from pants.backend.python.dependency_inference.parse_python_dependencies import ( + ParsedPythonDependencies, +) +from pants.backend.python.dependency_inference.parse_python_dependencies import ( + ParsedPythonImportInfo as ImpInfo, +) +from pants.backend.python.dependency_inference.parse_python_dependencies import ( + ParsePythonDependenciesRequest, +) +from pants.backend.python.framework.django import dependency_inference, detect_apps +from pants.backend.python.target_types import PythonSourceField, PythonSourceTarget +from pants.backend.python.util_rules import pex +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.core.util_rules import stripped_source_files +from pants.engine.addresses import Address +from pants.testutil.python_interpreter_selection import ( + skip_unless_python27_present, + skip_unless_python37_present, + skip_unless_python38_present, + skip_unless_python39_present, +) +from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.strutil import softwrap + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *parse_python_dependencies.rules(), + *stripped_source_files.rules(), + *pex.rules(), + *dependency_inference.rules(), + *detect_apps.rules(), + QueryRule(ParsedPythonDependencies, [ParsePythonDependenciesRequest]), + ], + target_types=[PythonSourceTarget], + ) + + +def assert_deps_parsed( + rule_runner: RuleRunner, + content: str, + *, + constraints: str, + expected_imports: dict[str, ImpInfo] | None = None, + expected_assets: list[str] | None = None, + filename: str = "path/to/app0/migrations/0001_initial.py", +) -> None: + expected_imports = expected_imports or {} + expected_assets = expected_assets or [] + rule_runner.set_options( + [ + "--no-python-infer-string-imports", + "--no-python-infer-assets", + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + rule_runner.write_files( + { + "BUILD": f"python_source(name='t', source={repr(filename)})", + "path/to/app1/BUILD": softwrap( + f"""\ + python_source( + source="apps.py", + interpreter_constraints=['{constraints}'], + ) + """ + ), + "path/to/app1/apps.py": softwrap( + """\ + class App1AppConfig(AppConfig): + name = "path.to.app1" + label = "app1" + """ + ), + "another/path/app2/BUILD": softwrap( + f"""\ + python_source( + source="apps.py", + interpreter_constraints=['{constraints}'], + ) + """ + ), + "another/path/app2/apps.py": softwrap( + """\ + class App2AppConfig(AppConfig): + name = "another.path.app2" + label = "app2_label" + """ + ), + filename: content, + } + ) + tgt = rule_runner.get_target(Address("", target_name="t")) + result = rule_runner.request( + ParsedPythonDependencies, + [ + ParsePythonDependenciesRequest( + tgt[PythonSourceField], + InterpreterConstraints([constraints]), + ) + ], + ) + assert dict(result.imports) == expected_imports + assert list(result.assets) == sorted(expected_assets) + + +def do_test_migration_dependencies(rule_runner: RuleRunner, constraints: str) -> None: + content = dedent( + """\ + class Migration(migrations.Migration): + dependencies = [ + ("app1", "0012_some_migration"), + ("app2_label", "0042_another_migration"), + ] + + operations = [] + """ + ) + assert_deps_parsed( + rule_runner, + content, + constraints=constraints, + expected_imports={ + "path.to.app1.migrations.0012_some_migration": ImpInfo(lineno=3, weak=True), + "another.path.app2.migrations.0042_another_migration": ImpInfo(lineno=4, weak=True), + }, + ) + + +@skip_unless_python27_present +def test_works_with_python2(rule_runner: RuleRunner) -> None: + do_test_migration_dependencies(rule_runner, constraints="CPython==2.7.*") + + +@skip_unless_python37_present +def test_works_with_python37(rule_runner: RuleRunner) -> None: + do_test_migration_dependencies(rule_runner, constraints="CPython==3.7.*") + + +@skip_unless_python38_present +def test_works_with_python38(rule_runner: RuleRunner) -> None: + do_test_migration_dependencies(rule_runner, constraints="CPython==3.8.*") + + +@skip_unless_python39_present +def test_works_with_python39(rule_runner: RuleRunner) -> None: + do_test_migration_dependencies(rule_runner, constraints="CPython==3.9.*") diff --git a/src/python/pants/backend/python/framework/django/detect_apps.py b/src/python/pants/backend/python/framework/django/detect_apps.py new file mode 100644 index 00000000000..a3e13a5a128 --- /dev/null +++ b/src/python/pants/backend/python/framework/django/detect_apps.py @@ -0,0 +1,128 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass + +from pants.backend.python.dependency_inference.parse_python_dependencies import get_scripts_digest +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import InterpreterConstraintsField +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.pex_environment import PythonExecutable +from pants.base.specs import FileGlobSpec, RawSpecs +from pants.engine.fs import AddPrefix, Digest, MergeDigests +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.process import Process, ProcessResult +from pants.engine.rules import collect_rules, rule +from pants.engine.target import ( + HydratedSources, + HydrateSourcesRequest, + SourcesField, + Target, + Targets, +) +from pants.util.frozendict import FrozenDict + + +@dataclass(frozen=True) +class DjangoAppsRequest: + pass + + +@dataclass(frozen=True) +class DjangoApps: + label_to_name: FrozenDict[str, str] + + def add_from_json(self, json_bytes: bytes) -> "DjangoApps": + apps = dict(self.label_to_name, **json.loads(json_bytes.decode())) + return DjangoApps(FrozenDict(sorted(apps.items()))) + + def to_json(self) -> bytes: + return json.dumps(dict(self.label_to_name), sort_keys=True).encode() + + +@rule +async def detect_django_apps( + _: DjangoAppsRequest, + python_setup: PythonSetup, +) -> DjangoApps: + # A Django app has a "name" - the full import path to the app ("path.to.myapp"), + # and a "label" - a short name, usually the last segment of the import path ("myapp"). + # + # An app provides this information via a subclass of AppConfig, living in a + # file named apps.py. Django loads this information into an app registry at runtime. + # + # Some parts of Django, notably migrations, use the label to reference apps. So to do custom + # Django dep inference, we need to know the label -> name mapping. + # + # The only truly correct way to enumerate Django apps is to run the Django app registry code. + # However we can't do this until after dep inference has completed, and even then it would be + # complicated: we wouldn't know which settings.py to use, or whether it's safe to run Django + # against that settings.py. Instead, we do this statically via parsing the apps.py file. + # + # NB: Legacy Django apps may not have an apps.py, in which case the label is assumed to be + # the name of the app dir, but the recommendation for many years has been to have it, and + # the Django startapp tool creates it for you. If an app does not have such an apps.py, + # then we won't be able to infer deps on that app unless we find other ways of detecting it. + # We should only do that if that case turns out to be common, and for some reason users can't + # simply create an apps.py to fix the issue. + # + # NB: Right now we only detect first-party apps in repo. We assume that third-party apps will + # be dep-inferred as a whole via the full package path in settings.py anyway. + # In the future we may find a way to map third-party apps here as well. + django_apps = DjangoApps(FrozenDict()) + targets = await Get( + Targets, + RawSpecs, + RawSpecs.create( + specs=[FileGlobSpec("**/apps.py")], description_of_origin="Django app detection" + ), + ) + if not targets: + return django_apps + + script_digest = await get_scripts_digest( + "pants.backend.python.framework.django.scripts", ["app_detector.py"] + ) + apps_sandbox_prefix = "_apps_to_detect" + + # Partition by ICs, so we can run the detector on the appropriate interpreter. + ics_to_tgts: dict[InterpreterConstraints, list[Target]] = defaultdict(list) + for tgt in targets: + ics = InterpreterConstraints( + tgt[InterpreterConstraintsField].value_or_global_default(python_setup) + ) + ics_to_tgts[ics].append(tgt) + + for ics, tgts in ics_to_tgts.items(): + sources = await MultiGet( + [Get(HydratedSources, HydrateSourcesRequest(tgt[SourcesField])) for tgt in tgts] + ) + apps_digest = await Get(Digest, MergeDigests([src.snapshot.digest for src in sources])) + prefixed_apps_digest = await Get(Digest, AddPrefix(apps_digest, apps_sandbox_prefix)) + + input_digest = await Get(Digest, MergeDigests([prefixed_apps_digest, script_digest])) + python_interpreter = await Get(PythonExecutable, InterpreterConstraints, ics) + + process_result = await Get( + ProcessResult, + Process( + argv=[ + python_interpreter.path, + "pants/backend/python/framework/django/scripts/app_detector.py", + apps_sandbox_prefix, + ], + input_digest=input_digest, + description="Detect Django apps", + ), + ) + django_apps = django_apps.add_from_json(process_result.stdout or b"{}") + return django_apps + + +def rules(): + return [ + *collect_rules(), + ] diff --git a/src/python/pants/backend/python/framework/django/detect_apps_test.py b/src/python/pants/backend/python/framework/django/detect_apps_test.py new file mode 100644 index 00000000000..b3b427abc4c --- /dev/null +++ b/src/python/pants/backend/python/framework/django/detect_apps_test.py @@ -0,0 +1,144 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import pytest + +from pants.backend.python.dependency_inference import parse_python_dependencies +from pants.backend.python.framework.django import dependency_inference, detect_apps +from pants.backend.python.framework.django.detect_apps import DjangoApps, DjangoAppsRequest +from pants.backend.python.target_types import PythonSourceTarget +from pants.backend.python.util_rules import pex +from pants.core.util_rules import stripped_source_files +from pants.engine.environment import EnvironmentName +from pants.testutil.python_interpreter_selection import ( + PY_27, + PY_39, + skip_unless_all_pythons_present, + skip_unless_python27_present, + skip_unless_python37_present, + skip_unless_python38_present, + skip_unless_python39_present, +) +from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.frozendict import FrozenDict +from pants.util.strutil import softwrap + + +@pytest.fixture +def rule_runner() -> RuleRunner: + ret = RuleRunner( + rules=[ + *parse_python_dependencies.rules(), + *stripped_source_files.rules(), + *pex.rules(), + *dependency_inference.rules(), + *detect_apps.rules(), + QueryRule(DjangoApps, [DjangoAppsRequest, EnvironmentName]), + ], + target_types=[PythonSourceTarget], + ) + ret.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return ret + + +def assert_apps_detected( + rule_runner: RuleRunner, + constraints1: str, + constraints2: str = "", +) -> None: + constraints2 = constraints2 or constraints1 + if "3." in constraints1: + py3_only = "async def i_will_parse_on_python3_not_on_python2():\n pass" + else: + py3_only = "" + if "2.7" in constraints2: + py2_only = "print 'I will parse on Python 2.7, not on Python 3'" + else: + py2_only = "" + rule_runner.write_files( + { + "path/to/app1/BUILD": softwrap( + f"""\ + {py3_only} + + python_source( + source="apps.py", + interpreter_constraints=['{constraints1}'], + ) + """ + ), + "path/to/app1/apps.py": softwrap( + """\ + class App1AppConfig(AppConfig): + name = "path.to.app1" + label = "app1" + """ + ), + "another/path/app2/BUILD": softwrap( + f"""\ + python_source( + source="apps.py", + interpreter_constraints=['{constraints2}'], + ) + """ + ), + "another/path/app2/apps.py": softwrap( + f"""\ + {py2_only} + + class App2AppConfig(AppConfig): + name = "another.path.app2" + label = "app2_label" + """ + ), + "some/app3/BUILD": softwrap( + f"""\ + python_source( + source="apps.py", + interpreter_constraints=['{constraints1}'], + ) + """ + ), + "some/app3/apps.py": softwrap( + """\ + class App3AppConfig(AppConfig): + name = "some.app3" + # No explicit label, should default to app3. + """ + ), + } + ) + result = rule_runner.request( + DjangoApps, + [DjangoAppsRequest()], + ) + assert result == DjangoApps( + FrozenDict({"app1": "path.to.app1", "app2_label": "another.path.app2", "app3": "some.app3"}) + ) + + +@skip_unless_python27_present +def test_works_with_python2(rule_runner: RuleRunner) -> None: + assert_apps_detected(rule_runner, constraints1="CPython==2.7.*") + + +@skip_unless_python37_present +def test_works_with_python37(rule_runner: RuleRunner) -> None: + assert_apps_detected(rule_runner, constraints1="CPython==3.7.*") + + +@skip_unless_python38_present +def test_works_with_python38(rule_runner: RuleRunner) -> None: + assert_apps_detected(rule_runner, constraints1="CPython==3.8.*") + + +@skip_unless_python39_present +def test_works_with_python39(rule_runner: RuleRunner) -> None: + assert_apps_detected(rule_runner, constraints1="CPython==3.9.*") + + +@skip_unless_all_pythons_present(PY_27, PY_39) +def test_partitioning_by_ics(rule_runner: RuleRunner) -> None: + assert_apps_detected(rule_runner, constraints1="CPython==3.9.*", constraints2="CPython==2.7.*") diff --git a/src/python/pants/backend/python/framework/django/scripts/BUILD b/src/python/pants/backend/python/framework/django/scripts/BUILD new file mode 100644 index 00000000000..68aaac88424 --- /dev/null +++ b/src/python/pants/backend/python/framework/django/scripts/BUILD @@ -0,0 +1,4 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/src/python/pants/backend/python/framework/django/scripts/__init__.py b/src/python/pants/backend/python/framework/django/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/framework/django/scripts/app_detector.py b/src/python/pants/backend/python/framework/django/scripts/app_detector.py new file mode 100644 index 00000000000..563f76c3290 --- /dev/null +++ b/src/python/pants/backend/python/framework/django/scripts/app_detector.py @@ -0,0 +1,99 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +# -*- coding: utf-8 -*- + +# NB: This must be compatible with Python 2.7 and 3.5+. +# NB: An easy way to debug this is to just invoke it on a file. +# E.g. +# $ python src/python/pants/backend/python/framework/django/scripts/app_detector.py PATHS +# Or +# $ ./pants run src/python/pants/backend/python/framework/django/scripts/app_detector.py -- PATHS + +from __future__ import print_function, unicode_literals + +import ast +import json +import os +import sys + + +class DjangoAppDetector(ast.NodeVisitor): + @staticmethod + def maybe_str(node): + if sys.version_info[0:2] < (3, 8): + return node.s if isinstance(node, ast.Str) else None + else: + return node.value if isinstance(node, ast.Constant) else None + + def __init__(self): + self._app_name = "" + self._app_label = "" + + @property + def app_name(self): + return self._app_name + + @property + def app_label(self): + return self._app_label or self._app_name.rpartition(".")[2] + + def visit_ClassDef(self, node): + # We detect an AppConfig subclass via the following heuristics: + # A) The definition is exactly `MyClassName(AppConfig)` (rather than + # `MyClassName(apps.AppConfig)` and so on). This is what Django's + # startapp tool generates, and is a very strong convention. + # - or - + # B) The class name ends with `AppConfig`. + # This catches violations of the conventions of A), e.g., if there + # are custom intermediate subclasses of AppConfig, or custom extra + # base classes. + # + # These should catch every non-perverse case in practice. + if node.name.endswith("AppConfig") or ( + len(node.bases) == 1 + and isinstance(node.bases[0], ast.Name) + and node.bases[0].id == "AppConfig" + ): + for child in node.body: + if isinstance(child, ast.Assign) and len(child.targets) == 1: + node_id = child.targets[0].id + if node_id == "name": + self._app_name = self.maybe_str(child.value) + elif node_id == "label": + self._app_label = self.maybe_str(child.value) + + +def handle_file(apps, path): + if os.path.basename(path) != "apps.py": + return + with open(path, "rb") as f: + content = f.read() + try: + tree = ast.parse(content, filename=path) + except SyntaxError: + return + + visitor = DjangoAppDetector() + visitor.visit(tree) + + if visitor.app_name: + apps[visitor.app_label] = visitor.app_name + + +def main(paths): + apps = {} # label -> full Python module path + for path in paths: + if os.path.isfile(path): + handle_file(apps, path) + elif os.path.isdir(path): + for dirpath, _, filenames in os.walk(path): + for filename in filenames: + handle_file(apps, os.path.join(dirpath, filename)) + + # We have to be careful to set the encoding explicitly and write raw bytes ourselves. + buffer = sys.stdout if sys.version_info[0:2] == (2, 7) else sys.stdout.buffer + buffer.write(json.dumps(apps).encode("utf8")) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/src/python/pants/backend/python/framework/django/scripts/dependency_visitor.py b/src/python/pants/backend/python/framework/django/scripts/dependency_visitor.py new file mode 100644 index 00000000000..f3082340b51 --- /dev/null +++ b/src/python/pants/backend/python/framework/django/scripts/dependency_visitor.py @@ -0,0 +1,55 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +# -*- coding: utf-8 -*- + +# NB: This must be compatible with Python 2.7 and 3.5+. +# NB: Expects a json file at "./apps.json" mapping Django app labels to their package paths. +# NB: An easy way to debug this is to invoke it directly on a file. +# E.g. +# $ PYTHONPATH=src/python VISITOR_CLASSNAMES=pants.backend.python.framework.django.scripts\ +# .dependency_visitor.DjangoDependencyVisitor \ +# python src/python/pants/backend/python/dependency_inference/scripts/main.py FILE + +from __future__ import print_function, unicode_literals + +import ast +import json + +from pants.backend.python.dependency_inference.scripts.dependency_visitor_base import ( + DependencyVisitorBase, +) + + +class DjangoDependencyVisitor(DependencyVisitorBase): + def __init__(self, *args, **kwargs): + super(DjangoDependencyVisitor, self).__init__(*args, **kwargs) + with open("apps.json", "r") as fp: + self._apps = json.load(fp) + + def visit_ClassDef(self, node): + # Detect `class Migration(migrations.Migration):` + if ( + node.name == "Migration" + and len(node.bases) > 0 + and node.bases[0].value.id == "migrations" + and node.bases[0].attr == "Migration" + ): + # Detect `dependencies = [("app1", "migration1"), ("app2", "migration2")]` + for child in node.body: + if ( + isinstance(child, ast.Assign) + and len(child.targets) == 1 + and child.targets[0].id == "dependencies" + and isinstance(child.value, (ast.Tuple, ast.List)) + ): + for elt in child.value.elts: + if isinstance(elt, (ast.Tuple, ast.List)) and len(elt.elts) == 2: + app = self.maybe_str(elt.elts[0]) + migration = self.maybe_str(elt.elts[1]) + if app is not None and migration is not None: + pkg = self._apps.get(app) + if pkg: + module = "{}.migrations.{}".format(pkg, migration) + self.add_weak_import(module, elt.lineno) + + self.generic_visit(node) diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints.py b/src/python/pants/backend/python/util_rules/interpreter_constraints.py index d7c393f16bc..b6fdf9477ef 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints.py @@ -149,13 +149,6 @@ def and_constraints(parsed_constraints: Sequence[Requirement]) -> Requirement: formatted_specs = ",".join(f"{op}{version}" for op, version in merged_specs) return parse_constraint(f"{expected_interpreter}{formatted_specs}") - def cmp_constraints(req1: Requirement, req2: Requirement) -> int: - if req1.project_name != req2.project_name: - return -1 if req1.project_name < req2.project_name else 1 - if req1.specs == req2.specs: - return 0 - return -1 if req1.specs < req2.specs else 1 - ored_constraints = ( and_constraints(constraints_product) for constraints_product in itertools.product(*parsed_constraint_sets)