From e227ba94f3db62c7017130611ca8a876608342e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 8 Oct 2021 14:42:01 +0200 Subject: [PATCH] Favor the "venv" sysconfig install scheme over the default and distutils scheme Python is preparing to allow re-distributors to set custom sysconfig install schemes in 3.11+: https://bugs.python.org/issue43976 Fedora is already adapting the default installation scheme to their needs: https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/AAGUFQZ4RZDU7KUN4HA43KQJCMSFR3GW/ With either of the above, the distributors need to signalize the paths used in virtual environments somehow. When they set the "venv" install scheme in sysconfig, it is now favored over the default sysconfig scheme as well as over distutils. A similar technique was proposed to Python, for the venv module: https://bugs.python.org/issue45413 Fixes https://github.com/pypa/virtualenv/issues/2208 --- docs/changelog/2208.feature.rst | 4 ++ src/virtualenv/discovery/py_info.py | 16 ++++- tests/unit/discovery/py_info/test_py_info.py | 61 ++++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/2208.feature.rst diff --git a/docs/changelog/2208.feature.rst b/docs/changelog/2208.feature.rst new file mode 100644 index 000000000..dbfd59a64 --- /dev/null +++ b/docs/changelog/2208.feature.rst @@ -0,0 +1,4 @@ +If a ``"venv"`` install scheme exists in ``sysconfig``, virtualenv now uses it to create new virtual environments. +This allows Python distributors, such as Fedora, to patch/replace the default install scheme without affecting +the paths in new virtual environments. +A similar technique `was proposed to Python, for the venv module `_ - by ``hroncok`` diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 0de612814..868173511 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -73,7 +73,18 @@ def abs_path(v): self.file_system_encoding = u(sys.getfilesystemencoding()) self.stdout_encoding = u(getattr(sys.stdout, "encoding", None)) - self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()} + if "venv" in sysconfig.get_scheme_names(): + self.sysconfig_scheme = "venv" + self.sysconfig_paths = { + u(i): u(sysconfig.get_path(i, expand=False, scheme="venv")) for i in sysconfig.get_path_names() + } + # we cannot use distutils at all if "venv" exists, distutils don't know it + self.distutils_install = {} + else: + self.sysconfig_scheme = None + self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()} + self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()} + # https://bugs.python.org/issue22199 makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) self.sysconfig = { @@ -95,7 +106,6 @@ def abs_path(v): if self.implementation == "PyPy" and sys.version_info.major == 2: self.sysconfig_vars[u"implementation_lower"] = u"python" - self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()} confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()} self.system_stdlib = self.sysconfig_path("stdlib", confs) self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) @@ -119,7 +129,7 @@ def _fast_get_system_executable(self): def install_path(self, key): result = self.distutils_install.get(key) - if result is None: # use sysconfig if distutils is unavailable + if result is None: # use sysconfig if sysconfig_scheme is set or distutils is unavailable # set prefixes to empty => result is relative from cwd prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()} diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index a0b160cb3..9984b91e2 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -6,6 +6,7 @@ import logging import os import sys +import sysconfig from collections import namedtuple from textwrap import dedent @@ -311,3 +312,63 @@ def test_py_info_to_system_raises(session_app_data, mocker, caplog, skip_if_test assert log.levelno == logging.INFO expected = "ignore {} due cannot resolve system due to RuntimeError('failed to detect ".format(sys.executable) assert expected in log.message + + +def _stringify_schemes_dict(schemes_dict): + """ + Since this file has from __future__ import unicode_literals, + we manually cast all values of mocked install_schemes to str() + as the original schemes are not unicode on Python 2. + """ + return {str(n): {str(k): str(v) for k, v in s.items()} for n, s in schemes_dict.items()} + + +def test_custom_venv_install_scheme_is_prefered(mocker): + # The paths in this test are Fedora paths, but we set them for nt as well, + # so the test also works on Windows, despite the actual values are nonsense there. + # Values were simplified to be compatible with all the supported Python versions. + default_scheme = { + "stdlib": "{base}/lib/python{py_version_short}", + "platstdlib": "{platbase}/lib/python{py_version_short}", + "purelib": "{base}/local/lib/python{py_version_short}/site-packages", + "platlib": "{platbase}/local/lib/python{py_version_short}/site-packages", + "include": "{base}/include/python{py_version_short}", + "platinclude": "{platbase}/include/python{py_version_short}", + "scripts": "{base}/local/bin", + "data": "{base}/local", + } + venv_scheme = {key: path.replace("local", "") for key, path in default_scheme.items()} + sysconfig_install_schemes = { + "posix_prefix": default_scheme, + "nt": default_scheme, + "venv": venv_scheme, + } + if getattr(sysconfig, "get_preferred_scheme", None): + sysconfig_install_schemes[sysconfig.get_preferred_scheme("prefix")] = default_scheme + + if sys.version_info[0] == 2: + sysconfig_install_schemes = _stringify_schemes_dict(sysconfig_install_schemes) + mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes) + + # On Python < 3.10, the distutils schemes are not derived from sysconfig schemes + # So we mock them as well to assert the custom "venv" install scheme has priority + distutils_scheme = { + "purelib": "$base/local/lib/python$py_version_short/site-packages", + "platlib": "$platbase/local/lib/python$py_version_short/site-packages", + "headers": "$base/include/python$py_version_short/$dist_name", + "scripts": "$base/local/bin", + "data": "$base/local", + } + distutils_schemes = { + "unix_prefix": distutils_scheme, + "nt": distutils_scheme, + } + + if sys.version_info[0] == 2: + distutils_schemes = _stringify_schemes_dict(distutils_schemes) + mocker.patch("distutils.command.install.INSTALL_SCHEMES", distutils_schemes) + + pyinfo = PythonInfo() + pyver = "{}.{}".format(pyinfo.version_info.major, pyinfo.version_info.minor) + assert pyinfo.install_path("scripts") == "bin" + assert pyinfo.install_path("purelib").replace(os.sep, "/") == "lib/python{}/site-packages".format(pyver)