Implement sysconfig-based location inference

news/9617.process.rst

@@ -0,0 +1,3 @@
+Start installation scheme migration from ``distutils`` to ``sysconfig``. A
+warning is implemented to detect differences between the two implementations to
+encourage user reports, so we can avoid breakages before they happen.

src/pip/_internal/build_env.py

@@ -6,7 +6,6 @@
 import sys
 import textwrap
 from collections import OrderedDict
-from distutils.sysconfig import get_python_lib
 from sysconfig import get_paths
 from types import TracebackType
 from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type
@@ -15,6 +14,7 @@
 
 from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
+from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
 
@@ -34,14 +34,7 @@ def __init__(self, path):
             'nt' if os.name == 'nt' else 'posix_prefix',
             vars={'base': path, 'platbase': path}
         )['scripts']
-        # Note: prefer distutils' sysconfig to get the
-        # library paths so PyPy is correctly supported.
-        purelib = get_python_lib(plat_specific=False, prefix=path)
-        platlib = get_python_lib(plat_specific=True, prefix=path)
-        if purelib == platlib:
-            self.lib_dirs = [purelib]
-        else:
-            self.lib_dirs = [purelib, platlib]
+        self.lib_dirs = get_prefixed_libs(path)
 
 
 class BuildEnvironment:
@@ -69,10 +62,7 @@ def __init__(self):
         # - ensure .pth files are honored
         # - prevent access to system site packages
         system_sites = {
-            os.path.normcase(site) for site in (
-                get_python_lib(plat_specific=False),
-                get_python_lib(plat_specific=True),
-            )
+            os.path.normcase(site) for site in (get_purelib(), get_platlib())
         }
         self._site_dir = os.path.join(temp_dir.path, 'site')
         if not os.path.exists(self._site_dir):

src/pip/_internal/commands/install.py

@@ -15,7 +15,7 @@
 from pip._internal.cli.req_command import RequirementCommand, with_cleanup
 from pip._internal.cli.status_codes import ERROR, SUCCESS
 from pip._internal.exceptions import CommandError, InstallationError
-from pip._internal.locations import distutils_scheme
+from pip._internal.locations import get_scheme
 from pip._internal.metadata import get_environment
 from pip._internal.models.format_control import FormatControl
 from pip._internal.operations.check import ConflictDetails, check_install_conflicts
@@ -455,10 +455,10 @@ def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
 
         # Checking both purelib and platlib directories for installed
         # packages to be moved to target directory
-        scheme = distutils_scheme('', home=target_temp_dir.path)
-        purelib_dir = scheme['purelib']
-        platlib_dir = scheme['platlib']
-        data_dir = scheme['data']
+        scheme = get_scheme('', home=target_temp_dir.path)
+        purelib_dir = scheme.purelib
+        platlib_dir = scheme.platlib
+        data_dir = scheme.data
 
         if os.path.exists(purelib_dir):
             lib_dir_list.append(purelib_dir)
@@ -574,9 +574,15 @@ def get_lib_location_guesses(
         prefix=None  # type: Optional[str]
 ):
     # type:(...) -> List[str]
-    scheme = distutils_scheme('', user=user, home=home, root=root,
-                              isolated=isolated, prefix=prefix)
-    return [scheme['purelib'], scheme['platlib']]
+    scheme = get_scheme(
+        '',
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+    return [scheme.purelib, scheme.platlib]
 
 
 def site_packages_writable(root, isolated):

src/pip/_internal/exceptions.py

@@ -59,6 +59,21 @@ def __str__(self):
         )
 
 
+class UserInstallationInvalid(InstallationError):
+    """A --user install is requested on an environment without user site."""
+
+    def __str__(self):
+        # type: () -> str
+        return "User base directory is not specified"
+
+
+class InvalidSchemeCombination(InstallationError):
+    def __str__(self):
+        # type: () -> str
+        before = ", ".join(str(a) for a in self.args[:-1])
+        return f"Cannot set {before} and {self.args[-1]} together"
+
+
 class DistributionNotFound(InstallationError):
     """Raised when a distribution cannot be found to satisfy a requirement"""
 

src/pip/_internal/locations/__init__.py

@@ -0,0 +1,184 @@
+import logging
+import pathlib
+import sys
+import sysconfig
+from typing import List, Optional
+
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
+
+from . import _distutils, _sysconfig
+from .base import (
+    USER_CACHE_DIR,
+    get_major_minor_version,
+    get_src_prefix,
+    site_packages,
+    user_site,
+)
+
+__all__ = [
+    "USER_CACHE_DIR",
+    "get_bin_prefix",
+    "get_bin_user",
+    "get_major_minor_version",
+    "get_platlib",
+    "get_prefixed_libs",
+    "get_purelib",
+    "get_scheme",
+    "get_src_prefix",
+    "site_packages",
+    "user_site",
+]
+
+
+logger = logging.getLogger(__name__)
+
+
+def _default_base(*, user: bool) -> str:
+    if user:
+        base = sysconfig.get_config_var("userbase")
+    else:
+        base = sysconfig.get_config_var("base")
+    assert base is not None
+    return base
+
+
+def _warn_if_mismatch(old: pathlib.Path, new: pathlib.Path, *, key: str) -> bool:
+    if old == new:
+        return False
+    issue_url = "https://github.com/pypa/pip/issues/9617"
+    message = (
+        "Value for %s does not match. Please report this to <%s>"
+        "\ndistutils: %s"
+        "\nsysconfig: %s"
+    )
+    logger.warning(message, key, issue_url, old, new)
+    return True
+
+
+def _log_context(
+    *,
+    user: bool = False,
+    home: Optional[str] = None,
+    root: Optional[str] = None,
+    prefix: Optional[str] = None,
+) -> None:
+    message = (
+        "Additional context:" "\nuser = %r" "\nhome = %r" "\nroot = %r" "\nprefix = %r"
+    )
+    logger.warning(message, user, home, root, prefix)
+
+
+def get_scheme(
+    dist_name,  # type: str
+    user=False,  # type: bool
+    home=None,  # type: Optional[str]
+    root=None,  # type: Optional[str]
+    isolated=False,  # type: bool
+    prefix=None,  # type: Optional[str]
+):
+    # type: (...) -> Scheme
+    old = _distutils.get_scheme(
+        dist_name,
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+    new = _sysconfig.get_scheme(
+        dist_name,
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+
+    base = prefix or home or _default_base(user=user)
+    warned = []
+    for k in SCHEME_KEYS:
+        # Extra join because distutils can return relative paths.
+        old_v = pathlib.Path(base, getattr(old, k))
+        new_v = pathlib.Path(getattr(new, k))
+
+        # distutils incorrectly put PyPy packages under ``site-packages/python``
+        # in the ``posix_home`` scheme, but PyPy devs said they expect the
+        # directory name to be ``pypy`` instead. So we treat this as a bug fix
+        # and not warn about it. See bpo-43307 and python/cpython#24628.
+        skip_pypy_special_case = (
+            sys.implementation.name == "pypy"
+            and home is not None
+            and k in ("platlib", "purelib")
+            and old_v.parent == new_v.parent
+            and old_v.name == "python"
+            and new_v.name == "pypy"
+        )
+        if skip_pypy_special_case:
+            continue
+
+        warned.append(_warn_if_mismatch(old_v, new_v, key=f"scheme.{k}"))
+
+    if any(warned):
+        _log_context(user=user, home=home, root=root, prefix=prefix)
+
+    return old
+
+
+def get_bin_prefix():
+    # type: () -> str
+    old = _distutils.get_bin_prefix()
+    new = _sysconfig.get_bin_prefix()
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
+        _log_context()
+    return old
+
+
+def get_bin_user():
+    # type: () -> str
+    return _sysconfig.get_scheme("", user=True).scripts
+
+
+def get_purelib():
+    # type: () -> str
+    """Return the default pure-Python lib location."""
+    old = _distutils.get_purelib()
+    new = _sysconfig.get_purelib()
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"):
+        _log_context()
+    return old
+
+
+def get_platlib():
+    # type: () -> str
+    """Return the default platform-shared lib location."""
+    old = _distutils.get_platlib()
+    new = _sysconfig.get_platlib()
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
+        _log_context()
+    return old
+
+
+def get_prefixed_libs(prefix):
+    # type: (str) -> List[str]
+    """Return the lib locations under ``prefix``."""
+    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
+    new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
+
+    warned = [
+        _warn_if_mismatch(
+            pathlib.Path(old_pure),
+            pathlib.Path(new_pure),
+            key="prefixed-purelib",
+        ),
+        _warn_if_mismatch(
+            pathlib.Path(old_plat),
+            pathlib.Path(new_plat),
+            key="prefixed-platlib",
+        ),
+    ]
+    if any(warned):
+        _log_context(prefix=prefix)
+
+    if old_pure == old_plat:
+        return [old_pure]
+    return [old_pure, old_plat]