diff --git a/news/5702.bugfix b/news/5702.bugfix new file mode 100644 index 00000000000..2541d745ed7 --- /dev/null +++ b/news/5702.bugfix @@ -0,0 +1 @@ +Correctly handle system site-packages, in virtual environments created with venv (PEP 405). diff --git a/news/7155.bugfix b/news/7155.bugfix new file mode 100644 index 00000000000..2541d745ed7 --- /dev/null +++ b/news/7155.bugfix @@ -0,0 +1 @@ +Correctly handle system site-packages, in virtual environments created with venv (PEP 405). diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py index 380db1c3281..d81e6ac54bb 100644 --- a/src/pip/_internal/utils/virtualenv.py +++ b/src/pip/_internal/utils/virtualenv.py @@ -1,34 +1,115 @@ -import os.path +from __future__ import absolute_import + +import logging +import os +import re import site import sys +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + +logger = logging.getLogger(__name__) +_INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( + r"include-system-site-packages\s*=\s*(?Ptrue|false)" +) + + +def _running_under_venv(): + # type: () -> bool + """Checks if sys.base_prefix and sys.prefix match. + + This handles PEP 405 compliant virtual environments. + """ + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + +def _running_under_regular_virtualenv(): + # type: () -> bool + """Checks if sys.real_prefix is set. + + This handles virtual environments created with pypa's virtualenv. + """ + # pypa/virtualenv case + return hasattr(sys, 'real_prefix') + def running_under_virtualenv(): # type: () -> bool + """Return True if we're running inside a virtualenv, False otherwise. """ - Return True if we're running inside a virtualenv, False otherwise. + return _running_under_venv() or _running_under_regular_virtualenv() + +def _get_pyvenv_cfg_lines(): + # type: () -> Optional[List[str]] + """Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines + + Returns None, if it could not read/access the file. """ - if hasattr(sys, 'real_prefix'): - # pypa/virtualenv case - return True - elif sys.prefix != getattr(sys, "base_prefix", sys.prefix): - # PEP 405 venv + pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') + try: + with open(pyvenv_cfg_file) as f: + return f.read().splitlines() # avoids trailing newlines + except IOError: + return None + + +def _no_global_under_venv(): + # type: () -> bool + """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion + + PEP 405 specifies that when system site-packages are not supposed to be + visible from a virtual environment, `pyvenv.cfg` must contain the following + line: + + include-system-site-packages = false + + Additionally, log a warning if accessing the file fails. + """ + cfg_lines = _get_pyvenv_cfg_lines() + if cfg_lines is None: + # We're not in a "sane" venv, so assume there is no system + # site-packages access (since that's PEP 405's default state). + logger.warning( + "Could not access 'pyvenv.cfg' despite a virtual environment " + "being active. Assuming global site-packages is not accessible " + "in this environment." + ) return True + for line in cfg_lines: + match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) + if match is not None and match.group('value') == 'false': + return True return False -def virtualenv_no_global(): +def _no_global_under_regular_virtualenv(): # type: () -> bool + """Check if "no-global-site-packages.txt" exists beside site.py + + This mirrors logic in pypa/virtualenv for determining whether system + site-packages are visible in the virtual environment. """ - Return True if in a venv and no system site packages. - """ - # this mirrors the logic in virtualenv.py for locating the - # no-global-site-packages.txt file site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) - no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') - if running_under_virtualenv() and os.path.isfile(no_global_file): - return True - else: - return False + no_global_site_packages_file = os.path.join( + site_mod_dir, 'no-global-site-packages.txt', + ) + return os.path.exists(no_global_site_packages_file) + + +def virtualenv_no_global(): + # type: () -> bool + """Returns a boolean, whether running in venv with no system site-packages. + """ + + if _running_under_regular_virtualenv(): + return _no_global_under_regular_virtualenv() + + if _running_under_venv(): + return _no_global_under_venv() + + return False diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index a4ff73d5c61..d13c931d0ef 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -703,6 +703,7 @@ def test_freeze_with_requirement_option_package_repeated_multi_file(script): @pytest.mark.network +@pytest.mark.incompatible_with_test_venv def test_freeze_user(script, virtualenv, data): """ Testing freeze with --user, first we have to install some stuff. @@ -733,6 +734,7 @@ def test_freeze_path(tmpdir, script, data): _check_output(result.stdout, expected) +@pytest.mark.incompatible_with_test_venv def test_freeze_path_exclude_user(tmpdir, script, data): """ Test freeze with --path and make sure packages from --user are not picked diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 3194b165384..364893eeca6 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -124,6 +124,7 @@ def test_pep518_allows_missing_requires(script, data, common_wheels): assert result.files_created +@pytest.mark.incompatible_with_test_venv def test_pep518_with_user_pip(script, pip_src, data, common_wheels): """ Check that build dependencies are installed into the build @@ -1593,6 +1594,7 @@ def test_target_install_ignores_distutils_config_install_prefix(script): assert relative_script_base not in result.files_created +@pytest.mark.incompatible_with_test_venv def test_user_config_accepted(script): # user set in the config file is parsed as 0/1 instead of True/False. # Check that this doesn't cause a problem. diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index a653e0b2fb2..2022e1fee53 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -227,6 +227,7 @@ def test_install_local_with_subdirectory(script): result.assert_installed('version_subpkg.py', editable=False) +@pytest.mark.incompatible_with_test_venv def test_wheel_user_with_prefix_in_pydistutils_cfg( script, data, with_wheel): if os.name == 'posix': diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 1ac644609e9..03725fc3229 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -27,6 +27,7 @@ def dist_in_site_packages(dist): class Tests_UserSite: @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_reset_env_system_site_packages_usersite(self, script): """ Check user site works as expected. @@ -42,6 +43,7 @@ def test_reset_env_system_site_packages_usersite(self, script): @pytest.mark.network @need_svn + @pytest.mark.incompatible_with_test_venv def test_install_subversion_usersite_editable_with_distribute( self, script, tmpdir): """ @@ -55,6 +57,7 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) + @pytest.mark.incompatible_with_test_venv def test_install_from_current_directory_into_usersite( self, script, data, with_wheel): """ @@ -75,7 +78,6 @@ def test_install_from_current_directory_into_usersite( ) assert dist_info_folder in result.files_created - @pytest.mark.incompatible_with_test_venv def test_install_user_venv_nositepkgs_fails(self, virtualenv, script, data): """ @@ -96,6 +98,7 @@ def test_install_user_venv_nositepkgs_fails(self, virtualenv, ) @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_usersite(self, script): """ Test user install with conflict in usersite updates usersite. @@ -119,6 +122,7 @@ def test_install_user_conflict_in_usersite(self, script): assert not isfile(initools_v3_file), initools_v3_file @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite(self, virtualenv, script): """ Test user install with conflict in global site ignores site and @@ -149,6 +153,7 @@ def test_install_user_conflict_in_globalsite(self, virtualenv, script): assert isdir(initools_folder) @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script): """ Test user install/upgrade with conflict in global site ignores site and @@ -178,6 +183,7 @@ def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script): assert isdir(initools_folder) @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite_and_usersite( self, virtualenv, script): """ @@ -214,6 +220,7 @@ def test_install_user_conflict_in_globalsite_and_usersite( assert isdir(initools_folder) @pytest.mark.network + @pytest.mark.incompatible_with_test_venv def test_install_user_in_global_virtualenv_with_conflict_fails( self, script): """ diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index 2dfec01eedb..e218d6e7f17 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -233,6 +233,7 @@ def test_wheel_record_lines_in_deterministic_order(script, data): assert record_lines == sorted(record_lines) +@pytest.mark.incompatible_with_test_venv def test_install_user_wheel(script, data, with_wheel): """ Test user install from wheel (that has a script) diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index a863c42c91a..53f4152c2b7 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -94,6 +94,7 @@ def test_local_columns_flag(simple_script): @pytest.mark.network +@pytest.mark.incompatible_with_test_venv def test_user_flag(script, data): """ Test the behavior of --user flag in the list command @@ -110,6 +111,7 @@ def test_user_flag(script, data): @pytest.mark.network +@pytest.mark.incompatible_with_test_venv def test_user_columns_flag(script, data): """ Test the behavior of --user --format=columns flags in the list command @@ -502,6 +504,7 @@ def test_list_path(tmpdir, script, data): assert {'name': 'simple', 'version': '2.0'} in json_result +@pytest.mark.incompatible_with_test_venv def test_list_path_exclude_user(tmpdir, script, data): """ Test list with --path and make sure packages from --user are not picked diff --git a/tests/functional/test_uninstall_user.py b/tests/functional/test_uninstall_user.py index f99f3f21c7d..58079a293a8 100644 --- a/tests/functional/test_uninstall_user.py +++ b/tests/functional/test_uninstall_user.py @@ -9,6 +9,7 @@ from tests.lib import assert_all_changes, pyversion +@pytest.mark.incompatible_with_test_venv class Tests_UninstallUserSite: @pytest.mark.network diff --git a/tests/unit/test_build_env.py b/tests/unit/test_build_env.py index 9db08a124dd..ff3b2e90cef 100644 --- a/tests/unit/test_build_env.py +++ b/tests/unit/test_build_env.py @@ -162,6 +162,7 @@ def test_build_env_overlay_prefix_has_priority(script): assert result.stdout.strip() == '2.0', str(result) +@pytest.mark.incompatible_with_test_venv def test_build_env_isolation(script): # Create dummy `pkg` wheel. diff --git a/tests/unit/test_utils_virtualenv.py b/tests/unit/test_utils_virtualenv.py index 80e30404bda..625539d7617 100644 --- a/tests/unit/test_utils_virtualenv.py +++ b/tests/unit/test_utils_virtualenv.py @@ -1,3 +1,4 @@ +import logging import site import sys @@ -32,20 +33,103 @@ def test_running_under_virtualenv( @pytest.mark.parametrize( - "running_under_virtualenv, no_global_file, expected", [ + "under_virtualenv, no_global_file, expected", [ (False, False, False), (False, True, False), (True, False, False), (True, True, True), ], ) -def test_virtualenv_no_global( - monkeypatch, tmpdir, - running_under_virtualenv, no_global_file, expected): +def test_virtualenv_no_global_with_regular_virtualenv( + monkeypatch, + tmpdir, + under_virtualenv, + no_global_file, + expected, +): + monkeypatch.setattr(virtualenv, '_running_under_venv', lambda: False) + monkeypatch.setattr(site, '__file__', tmpdir / 'site.py') monkeypatch.setattr( - virtualenv, 'running_under_virtualenv', - lambda: running_under_virtualenv) + virtualenv, '_running_under_regular_virtualenv', + lambda: under_virtualenv, + ) if no_global_file: (tmpdir / 'no-global-site-packages.txt').touch() + assert virtualenv.virtualenv_no_global() == expected + + +@pytest.mark.parametrize( + "pyvenv_cfg_lines, under_venv, expected, expect_warning", [ + (None, False, False, False), + (None, True, True, True), # this has a warning. + ( + [ + "home = ", + "include-system-site-packages = true", + "version = ", + ], + True, + False, + False, + ), + ( + [ + "home = ", + "include-system-site-packages = false", + "version = ", + ], + True, + True, + False, + ), + ], +) +def test_virtualenv_no_global_with_pep_405_virtual_environment( + monkeypatch, + caplog, + pyvenv_cfg_lines, + under_venv, + expected, + expect_warning, +): + monkeypatch.setattr( + virtualenv, '_running_under_regular_virtualenv', lambda: False + ) + monkeypatch.setattr( + virtualenv, '_get_pyvenv_cfg_lines', lambda: pyvenv_cfg_lines + ) + monkeypatch.setattr(virtualenv, '_running_under_venv', lambda: under_venv) + + with caplog.at_level(logging.WARNING): + assert virtualenv.virtualenv_no_global() == expected + + if expect_warning: + assert caplog.records + + # Check for basic information + message = caplog.records[-1].getMessage().lower() + assert "could not access 'pyvenv.cfg'" in message + assert "assuming global site-packages is not accessible" in message + + +@pytest.mark.parametrize( + "contents, expected", [ + (None, None), + ("", []), + ("a = b\nc = d\n", ["a = b", "c = d"]), + ("a = b\nc = d", ["a = b", "c = d"]), # no trailing newlines + ] +) +def test_get_pyvenv_cfg_lines_for_pep_405_virtual_environment( + monkeypatch, + tmpdir, + contents, + expected, +): + monkeypatch.setattr(sys, 'prefix', str(tmpdir)) + if contents is not None: + tmpdir.joinpath('pyvenv.cfg').write_text(contents) + + assert virtualenv._get_pyvenv_cfg_lines() == expected