Ensure lockfile target exists before injecting a dependency to it. (Cherry-pick of #17365)

src/python/pants/backend/python/goals/lockfile.py

@@ -287,7 +287,7 @@ async def python_lockfile_synthetic_targets(
     request: PythonSyntheticLockfileTargetsRequest,
     python_setup: PythonSetup,
 ) -> SyntheticAddressMaps:
-    if not python_setup.enable_resolves:
+    if not python_setup.enable_synthetic_lockfiles:
         return SyntheticAddressMaps()
 
     resolves = [

src/python/pants/backend/python/macros/common_requirements_rule.py

@@ -0,0 +1,169 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import itertools
+import logging
+import os
+from typing import Callable, Iterable, cast
+
+from packaging.utils import canonicalize_name as canonicalize_project_name
+
+from pants.backend.python.macros.common_fields import (
+    ModuleMappingField,
+    TypeStubsModuleMappingField,
+)
+from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.target_types import (
+    PythonRequirementModulesField,
+    PythonRequirementResolveField,
+    PythonRequirementsField,
+    PythonRequirementTarget,
+    PythonRequirementTypeStubModulesField,
+)
+from pants.core.target_types import (
+    TargetGeneratorSourcesHelperSourcesField,
+    TargetGeneratorSourcesHelperTarget,
+)
+from pants.engine.addresses import Address
+from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
+from pants.engine.internals.target_adaptor import TargetAdaptor, TargetAdaptorRequest
+from pants.engine.rules import Get, rule_helper
+from pants.engine.target import (
+    Dependencies,
+    GenerateTargetsRequest,
+    InvalidFieldException,
+    SingleSourceField,
+)
+from pants.engine.unions import UnionMembership
+from pants.util.strutil import softwrap
+
+logger = logging.getLogger(__name__)
+ParseRequirementsCallback = Callable[[bytes, str], Iterable[PipRequirement]]
+
+
+@rule_helper
+async def _generate_requirements(
+    request: GenerateTargetsRequest,
+    union_membership: UnionMembership,
+    python_setup: PythonSetup,
+    parse_requirements_callback: ParseRequirementsCallback,
+) -> Iterable[PythonRequirementTarget]:
+    generator = request.generator
+    requirements_rel_path = generator[SingleSourceField].value
+    requirements_full_path = generator[SingleSourceField].file_path
+    overrides = {
+        canonicalize_project_name(k): v
+        for k, v in request.require_unparametrized_overrides().items()
+    }
+
+    # Pretend this is just another generated target, for typing purposes.
+    file_tgt = cast(
+        "PythonRequirementTarget",
+        TargetGeneratorSourcesHelperTarget(
+            {TargetGeneratorSourcesHelperSourcesField.alias: requirements_rel_path},
+            Address(
+                request.template_address.spec_path,
+                target_name=request.template_address.target_name,
+                relative_file_path=requirements_rel_path,
+            ),
+            union_membership,
+        ),
+    )
+
+    req_deps = [file_tgt.address.spec]
+
+    resolve = request.template.get(
+        PythonRequirementResolveField.alias, python_setup.default_resolve
+    )
+    lockfile = (
+        python_setup.resolves.get(resolve) if python_setup.enable_synthetic_lockfiles else None
+    )
+    if lockfile:
+        lockfile_address = Address(
+            os.path.dirname(lockfile),
+            target_name=resolve,
+        )
+        target_adaptor = await Get(
+            TargetAdaptor,
+            TargetAdaptorRequest(
+                description_of_origin=f"{generator.alias} lockfile dep for the {resolve} resolve",
+                address=lockfile_address,
+            ),
+        )
+        if target_adaptor.type_alias == "_lockfiles":
+            req_deps.append(f"{lockfile}:{resolve}")
+        else:
+            logger.warning(
+                softwrap(
+                    f"""
+                    The synthetic lockfile target for {lockfile} is being shadowed by the
+                    {target_adaptor.type_alias} target {lockfile_address}.
+
+                    There will not be any dependency to the lockfile.
+
+                    Resolve by either renaming the shadowing target, the resolve {resolve!r} or
+                    moving the target or the lockfile to another directory.
+                    """
+                )
+            )
+
+    digest_contents = await Get(
+        DigestContents,
+        PathGlobs(
+            [requirements_full_path],
+            glob_match_error_behavior=GlobMatchErrorBehavior.error,
+            description_of_origin=f"{generator}'s field `{SingleSourceField.alias}`",
+        ),
+    )
+
+    module_mapping = generator[ModuleMappingField].value
+    stubs_mapping = generator[TypeStubsModuleMappingField].value
+
+    def generate_tgt(
+        project_name: str, parsed_reqs: Iterable[PipRequirement]
+    ) -> PythonRequirementTarget:
+        normalized_proj_name = canonicalize_project_name(project_name)
+        tgt_overrides = overrides.pop(normalized_proj_name, {})
+        if Dependencies.alias in tgt_overrides:
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
+
+        return PythonRequirementTarget(
+            {
+                **request.template,
+                PythonRequirementsField.alias: list(parsed_reqs),
+                PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
+                PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
+                    normalized_proj_name
+                ),
+                # This may get overridden by `tgt_overrides`, which will have already added in
+                # the file tgt.
+                Dependencies.alias: req_deps,
+                **tgt_overrides,
+            },
+            request.template_address.create_generated(project_name),
+            union_membership,
+        )
+
+    requirements = parse_requirements_callback(digest_contents[0].content, requirements_full_path)
+    grouped_requirements = itertools.groupby(
+        requirements, lambda parsed_req: parsed_req.project_name
+    )
+    result = tuple(
+        generate_tgt(project_name, parsed_reqs_)
+        for project_name, parsed_reqs_ in grouped_requirements
+    ) + (file_tgt,)
+
+    if overrides:
+        raise InvalidFieldException(
+            softwrap(
+                f"""
+                Unused key in the `overrides` field for {request.template_address}:
+                {sorted(overrides)}
+                """
+            )
+        )
+
+    return result

src/python/pants/backend/python/macros/pipenv_requirements.py

@@ -5,41 +5,25 @@
 
 import json
 
-from packaging.utils import canonicalize_name as canonicalize_project_name
-
 from pants.backend.python.macros.common_fields import (
     ModuleMappingField,
     RequirementsOverrideField,
     TypeStubsModuleMappingField,
 )
+from pants.backend.python.macros.common_requirements_rule import _generate_requirements
 from pants.backend.python.pip_requirement import PipRequirement
 from pants.backend.python.subsystems.setup import PythonSetup
-from pants.backend.python.target_types import (
-    PythonRequirementModulesField,
-    PythonRequirementResolveField,
-    PythonRequirementsField,
-    PythonRequirementTarget,
-    PythonRequirementTypeStubModulesField,
-)
-from pants.core.target_types import (
-    TargetGeneratorSourcesHelperSourcesField,
-    TargetGeneratorSourcesHelperTarget,
-)
-from pants.engine.addresses import Address
-from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
-from pants.engine.rules import Get, collect_rules, rule
+from pants.backend.python.target_types import PythonRequirementResolveField, PythonRequirementTarget
+from pants.engine.rules import collect_rules, rule
 from pants.engine.target import (
     COMMON_TARGET_FIELDS,
-    Dependencies,
     GeneratedTargets,
     GenerateTargetsRequest,
-    InvalidFieldException,
     SingleSourceField,
     TargetGenerator,
 )
 from pants.engine.unions import UnionMembership, UnionRule
 from pants.util.logging import LogLevel
-from pants.util.strutil import softwrap
 
 
 class PipenvSourceField(SingleSourceField):
@@ -70,90 +54,23 @@ class GenerateFromPipenvRequirementsRequest(GenerateTargetsRequest):
 # TODO(#10655): add support for PEP 440 direct references (aka VCS style).
 # TODO(#10655): differentiate between Pipfile vs. Pipfile.lock.
 @rule(desc="Generate `python_requirement` targets from Pipfile.lock", level=LogLevel.DEBUG)
-async def generate_from_pipenv_requirement(
+async def generate_from_pipenv_requirements(
     request: GenerateFromPipenvRequirementsRequest,
     union_membership: UnionMembership,
     python_setup: PythonSetup,
 ) -> GeneratedTargets:
-    generator = request.generator
-    lock_rel_path = generator[PipenvSourceField].value
-    lock_full_path = generator[PipenvSourceField].file_path
-    overrides = {
-        canonicalize_project_name(k): v
-        for k, v in request.require_unparametrized_overrides().items()
-    }
-
-    file_tgt = TargetGeneratorSourcesHelperTarget(
-        {TargetGeneratorSourcesHelperSourcesField.alias: lock_rel_path},
-        Address(
-            request.template_address.spec_path,
-            target_name=request.template_address.target_name,
-            relative_file_path=lock_rel_path,
-        ),
+    result = await _generate_requirements(
+        request,
         union_membership,
+        python_setup,
+        parse_requirements_callback=parse_pipenv_requirements,
     )
+    return GeneratedTargets(request.generator, result)
 
-    req_deps = [file_tgt.address.spec]
-
-    resolve = request.template.get(
-        PythonRequirementResolveField.alias, python_setup.default_resolve
-    )
-    lockfile = python_setup.resolves.get(resolve) if python_setup.enable_resolves else None
-    if lockfile:
-        req_deps.append(f"{lockfile}:{resolve}")
-
-    digest_contents = await Get(
-        DigestContents,
-        PathGlobs(
-            [lock_full_path],
-            glob_match_error_behavior=GlobMatchErrorBehavior.error,
-            description_of_origin=f"{generator}'s field `{PipenvSourceField.alias}`",
-        ),
-    )
 
-    module_mapping = generator[ModuleMappingField].value
-    stubs_mapping = generator[TypeStubsModuleMappingField].value
-
-    def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
-        normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
-        tgt_overrides = overrides.pop(normalized_proj_name, {})
-        if Dependencies.alias in tgt_overrides:
-            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
-
-        return PythonRequirementTarget(
-            {
-                **request.template,
-                PythonRequirementsField.alias: [parsed_req],
-                PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
-                PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
-                    normalized_proj_name
-                ),
-                # This may get overridden by `tgt_overrides`, which will have already added in
-                # the file tgt.
-                Dependencies.alias: req_deps,
-                **tgt_overrides,
-            },
-            request.template_address.create_generated(parsed_req.project_name),
-            union_membership,
-        )
-
-    reqs = parse_pipenv_requirements(digest_contents[0].content)
-    result = tuple(generate_tgt(req) for req in reqs) + (file_tgt,)
-
-    if overrides:
-        raise InvalidFieldException(
-            softwrap(
-                f"""
-                Unused key in the `overrides` field for {request.template_address}:
-                {sorted(overrides)}
-                """
-            )
-        )
-
-    return GeneratedTargets(generator, result)
-
-
-def parse_pipenv_requirements(file_contents: bytes) -> tuple[PipRequirement, ...]:
+def parse_pipenv_requirements(
+    file_contents: bytes, file_path: str = ""
+) -> tuple[PipRequirement, ...]:
     lock_info = json.loads(file_contents)
 
     def _parse_pipenv_requirement(raw_req: str, info: dict) -> PipRequirement:

src/python/pants/backend/python/macros/pipenv_requirements_test.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from pants.backend.python.goals import lockfile
 from pants.backend.python.macros import pipenv_requirements
 from pants.backend.python.macros.pipenv_requirements import PipenvRequirementsTargetGenerator
 from pants.backend.python.target_types import PythonRequirementTarget
@@ -21,6 +22,7 @@
 def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=(
+            *lockfile.rules(),
             *pipenv_requirements.rules(),
             QueryRule(_TargetParametrizations, [_TargetParametrizationsRequest]),
         ),