Fix module_mapping to work regardless of capitalization and - vs _

src/python/pants/backend/python/dependency_inference/default_module_mapping.py

@@ -1,28 +1,23 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-# notice: using sets here to ensure the mapping is hashable
+# NB: You must use lowercase and replace all `_` and `.` with `-` for the requirement's name.
+# See https://www.python.org/dev/peps/pep-0503/#normalized-names.
 DEFAULT_MODULE_MAPPING = {
     "ansicolors": ("colors",),
+    "apache-airflow": ("airflow",),
     "attrs": ("attr",),
     "beautifulsoup4": ("bs4",),
     "djangorestframework": ("rest_framework",),
     "enum34": ("enum",),
-    "paho_mqtt": ("paho",),
+    "paho-mqtt": ("paho",),
     "protobuf": ("google.protobuf",),
     "pycrypto": ("Crypto",),
     "pyopenssl": ("OpenSSL",),
     "python-dateutil": ("dateutil",),
     "python-jose": ("jose",),
-    "PyYAML": ("yaml",),
-    "pymongo": (
-        "bson",
-        "gridfs",
-    ),
-    "pytest_runner": ("ptr",),
-    "python_dateutil": ("dateutil",),
-    "setuptools": (
-        "easy_install",
-        "pkg_resources",
-    ),
+    "pyyaml": ("yaml",),
+    "pymongo": ("bson", "gridfs"),
+    "pytest-runner": ("ptr",),
+    "setuptools": ("easy_install", "pkg_resources"),
 }

src/python/pants/backend/python/dependency_inference/module_mapper.py

@@ -9,6 +9,8 @@
 from pathlib import PurePath
 from typing import DefaultDict
 
+from packaging.utils import canonicalize_name as canonicalize_project_name
+
 from pants.backend.python.target_types import (
     ModuleMappingField,
     PythonRequirementsField,
@@ -238,10 +240,10 @@ async def map_third_party_modules_to_addresses() -> ThirdPartyPythonModuleMappin
         if not tgt.has_field(PythonRequirementsField):
             continue
         module_map = tgt.get(ModuleMappingField).value
-        for python_req in tgt[PythonRequirementsField].value:
+        for req in tgt[PythonRequirementsField].value:
+            normalized_project_name = canonicalize_project_name(req.project_name)
             modules = module_map.get(
-                python_req.project_name,
-                [python_req.project_name.lower().replace("-", "_")],
+                normalized_project_name, [normalized_project_name.replace("-", "_")]
             )
             for module in modules:
                 if module in modules_to_addresses:

src/python/pants/backend/python/dependency_inference/module_mapper_test.py

@@ -7,9 +7,11 @@
 from textwrap import dedent
 
 import pytest
+from packaging.utils import canonicalize_name as canonicalize_project_name
 
 from pants.backend.codegen.protobuf.python import python_protobuf_module_mapper
 from pants.backend.codegen.protobuf.target_types import ProtobufLibrary
+from pants.backend.python.dependency_inference.default_module_mapping import DEFAULT_MODULE_MAPPING
 from pants.backend.python.dependency_inference.module_mapper import (
     FirstPartyPythonModuleMapping,
     PythonModule,
@@ -24,6 +26,13 @@
 from pants.util.frozendict import FrozenDict
 
 
+def test_default_module_mapping_is_normalized() -> None:
+    for k in DEFAULT_MODULE_MAPPING:
+        assert k == canonicalize_project_name(
+            k
+        ), "Please update `DEFAULT_MODULE_MAPPING` to use canonical project names"
+
+
 @pytest.mark.parametrize(
     "stripped_path,expected",
     [
@@ -255,7 +264,8 @@ def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
 
             python_requirement_library(
               name='un_normalized',
-              requirements=['Un-Normalized-Project>3', 'two_owners'],
+              requirements=['Un-Normalized-Project>3', 'two_owners', 'DiFFerent-than_Mapping'],
+              module_mapping={"different_THAN-mapping": ["different_than_mapping"]},
             )
 
             python_requirement_library(
@@ -272,6 +282,7 @@ def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
         mapping=FrozenDict(
             {
                 "colors": Address("3rdparty/python", target_name="ansicolors"),
+                "different_than_mapping": Address("3rdparty/python", target_name="un_normalized"),
                 "local_dist": Address("3rdparty/python", target_name="direct_references"),
                 "pip": Address("3rdparty/python", target_name="direct_references"),
                 "req1": Address("3rdparty/python", target_name="req1"),

src/python/pants/backend/python/target_types.py

@@ -6,12 +6,14 @@
 import collections.abc
 import logging
 import os.path
+import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
 from textwrap import dedent
 from typing import Dict, Iterable, Iterator, Optional, Tuple, Union, cast
 
+from packaging.utils import canonicalize_name as canonicalize_project_name
 from pkg_resources import Requirement
 
 from pants.backend.python.dependency_inference.default_module_mapping import DEFAULT_MODULE_MAPPING
@@ -587,7 +589,7 @@ class ModuleMappingField(DictStringToStringSequenceField):
         "A mapping of requirement names to a list of the modules they provide.\n\nFor example, "
         '`{"ansicolors": ["colors"]}`. Any unspecified requirements will use the requirement '
         'name as the default module, e.g. "Django" will default to `["django"]`.\n\nThis is '
-        "used for Pants to be able to infer dependencies in BUILD files."
+        "used to infer dependencies."
     )
     value: FrozenDict[str, Tuple[str, ...]]
 
@@ -596,7 +598,12 @@ def compute_value(
         cls, raw_value: Optional[Dict[str, Iterable[str]]], address: Address
     ) -> FrozenDict[str, Tuple[str, ...]]:
         provided_mapping = super().compute_value(raw_value, address)
-        return FrozenDict({**DEFAULT_MODULE_MAPPING, **(provided_mapping or {})})
+        return FrozenDict(
+            {
+                **DEFAULT_MODULE_MAPPING,
+                **{canonicalize_project_name(k): v for k, v in (provided_mapping or {}).items()},
+            }
+        )
 
 
 class PythonRequirementLibrary(Target):