-
-
Notifications
You must be signed in to change notification settings - Fork 646
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Port Django migrations inference away from `PythonDependencyVisitorRe…
…quest` (#19008) This is the only in-repo plugin using `PythonDependencyVisitorRequest`, meaning when the rust parser is the new only implementation `PythonDependencyVisitorRequest` and plumbing can be removed (sooner if we refactor) Implementation: - Switched to running a dedicated process for scraping, instead of piggybacking off of the existing one - Only scrape from files directly under a `migrations` directory
- Loading branch information
1 parent
9d845f0
commit f9188ab
Showing
7 changed files
with
174 additions
and
119 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
115 changes: 94 additions & 21 deletions
115
src/python/pants/backend/python/framework/django/dependency_inference.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,49 +1,122 @@ | ||
| # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). | ||
| # Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
|
||
| from dataclasses import dataclass | ||
| import json | ||
| from pathlib import PurePath | ||
|
|
||
| from pants.backend.python.dependency_inference.parse_python_dependencies import ( | ||
| PythonDependencyVisitor, | ||
| PythonDependencyVisitorRequest, | ||
| get_scripts_digest, | ||
| ParsedPythonAssetPaths, | ||
| ParsedPythonDependencies, | ||
| ParsedPythonImportInfo, | ||
| ParsedPythonImports, | ||
| ) | ||
| from pants.backend.python.dependency_inference.rules import ( | ||
| ImportOwnerStatus, | ||
| PythonImportDependenciesInferenceFieldSet, | ||
| ResolvedParsedPythonDependencies, | ||
| ResolvedParsedPythonDependenciesRequest, | ||
| ) | ||
| from pants.backend.python.framework.django.detect_apps import DjangoApps | ||
| from pants.backend.python.subsystems.setup import PythonSetup | ||
| from pants.backend.python.target_types import EntryPoint | ||
| from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints | ||
| from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess | ||
| from pants.core.util_rules.source_files import SourceFilesRequest | ||
| from pants.core.util_rules.stripped_source_files import StrippedSourceFiles | ||
| from pants.engine.fs import CreateDigest, FileContent | ||
| from pants.engine.internals.native_engine import Digest, MergeDigests | ||
| from pants.engine.internals.native_engine import Digest | ||
| from pants.engine.internals.selectors import Get | ||
| from pants.engine.process import ProcessResult | ||
| from pants.engine.rules import collect_rules, rule | ||
| from pants.engine.target import InferDependenciesRequest, InferredDependencies | ||
| from pants.engine.unions import UnionRule | ||
| from pants.util.frozendict import FrozenDict | ||
| from pants.util.logging import LogLevel | ||
| from pants.util.resources import read_resource | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class DjangoDependencyVisitorRequest(PythonDependencyVisitorRequest): | ||
| pass | ||
| class InferDjangoDependencies(InferDependenciesRequest): | ||
| infer_from = PythonImportDependenciesInferenceFieldSet | ||
|
|
||
|
|
||
| _scripts_package = "pants.backend.python.framework.django.scripts" | ||
| _visitor_resource = "scripts/dependency_visitor.py" | ||
|
|
||
|
|
||
| @rule | ||
| async def django_parser_script( | ||
| _: DjangoDependencyVisitorRequest, | ||
| request: InferDjangoDependencies, | ||
| python_setup: PythonSetup, | ||
| django_apps: DjangoApps, | ||
| ) -> PythonDependencyVisitor: | ||
| django_apps_digest = await Get( | ||
| Digest, CreateDigest([FileContent("apps.json", django_apps.to_json())]) | ||
| ) -> InferredDependencies: | ||
| source_field = request.field_set.source | ||
| # NB: This doesn't consider https://docs.djangoproject.com/en/4.2/ref/settings/#std-setting-MIGRATION_MODULES | ||
| if not PurePath(source_field.file_path).match("migrations/*.py"): | ||
| return InferredDependencies([]) | ||
|
|
||
| stripped_sources = await Get( | ||
| StrippedSourceFiles, SourceFilesRequest([request.field_set.source]) | ||
| ) | ||
| assert len(stripped_sources.snapshot.files) == 1 | ||
|
|
||
| file_content = FileContent("__visitor.py", read_resource(__name__, _visitor_resource)) | ||
| visitor_digest = await Get(Digest, CreateDigest([file_content])) | ||
| venv_pex = await Get( | ||
| VenvPex, | ||
| PexRequest( | ||
| output_filename="__visitor.pex", | ||
| internal_only=True, | ||
| main=EntryPoint("__visitor"), | ||
| interpreter_constraints=InterpreterConstraints.create_from_compatibility_fields( | ||
| [request.field_set.interpreter_constraints], python_setup=python_setup | ||
| ), | ||
| sources=visitor_digest, | ||
| ), | ||
| ) | ||
| process_result = await Get( | ||
| ProcessResult, | ||
| VenvPexProcess( | ||
| venv_pex, | ||
| argv=[stripped_sources.snapshot.files[0]], | ||
| description=f"Determine Django app dependencies for {request.field_set.address}", | ||
| input_digest=stripped_sources.snapshot.digest, | ||
| level=LogLevel.DEBUG, | ||
| ), | ||
| ) | ||
| scripts_digest = await get_scripts_digest(_scripts_package, ["dependency_visitor.py"]) | ||
| digest = await Get(Digest, MergeDigests([django_apps_digest, scripts_digest])) | ||
| return PythonDependencyVisitor( | ||
| digest=digest, | ||
| classname=f"{_scripts_package}.dependency_visitor.DjangoDependencyVisitor", | ||
| env=FrozenDict({}), | ||
| # See in script for where we explicitly encoded as utf8. Even though utf8 is the | ||
| # default for decode(), we make that explicit here for emphasis. | ||
| process_output = process_result.stdout.decode("utf8") or "{}" | ||
| modules = [ | ||
| "{}.migrations.{}".format(django_apps.label_to_name[label], migration) | ||
| for label, migration in json.loads(process_output) | ||
| if label in django_apps.label_to_name | ||
| ] | ||
| resolve = request.field_set.resolve.normalized_value(python_setup) | ||
|
|
||
| resolved_dependencies = await Get( | ||
| ResolvedParsedPythonDependencies, | ||
| ResolvedParsedPythonDependenciesRequest( | ||
| request.field_set, | ||
| ParsedPythonDependencies( | ||
| ParsedPythonImports( | ||
| (module, ParsedPythonImportInfo(0, False)) for module in modules | ||
| ), | ||
| ParsedPythonAssetPaths(), | ||
| ), | ||
| resolve, | ||
| ), | ||
| ) | ||
|
|
||
| return InferredDependencies( | ||
| sorted( | ||
| address | ||
| for result in resolved_dependencies.resolve_results.values() | ||
| if result.status in (ImportOwnerStatus.unambiguous, ImportOwnerStatus.disambiguated) | ||
| for address in result.address | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| def rules(): | ||
| return [ | ||
| UnionRule(PythonDependencyVisitorRequest, DjangoDependencyVisitorRequest), | ||
| *collect_rules(), | ||
| UnionRule(InferDependenciesRequest, InferDjangoDependencies), | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.