From 0252c04a16cd93fe422cebf0b48453b559a2e404 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 7 Nov 2021 13:41:02 +0200 Subject: [PATCH 1/2] Drop support for soon-EOL Python 3.6 --- .github/workflows/ci.yml | 8 +- docs/html/development/ci.rst | 46 +++++----- docs/html/development/getting-started.rst | 16 ++-- docs/html/installation.md | 2 +- news/10641.removal.rst | 1 + noxfile.py | 2 +- setup.py | 3 +- src/pip/_internal/build_env.py | 9 +- src/pip/_internal/commands/list.py | 3 +- src/pip/_internal/utils/parallel.py | 103 ---------------------- tests/unit/test_utils_parallel.py | 73 --------------- tox.ini | 2 +- 12 files changed, 40 insertions(+), 228 deletions(-) create mode 100644 news/10641.removal.rst delete mode 100644 src/pip/_internal/utils/parallel.py delete mode 100644 tests/unit/test_utils_parallel.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51f7b8f9bbb..acad8f6fc4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,11 +102,10 @@ jobs: matrix: os: [Ubuntu, MacOS] python: - - 3.6 - 3.7 - 3.8 - 3.9 - - "3.10.0-alpha - 3.10" + - "3.10" steps: - uses: actions/checkout@v2 @@ -151,12 +150,11 @@ jobs: matrix: os: [Windows] python: - - 3.6 + - 3.7 # Commented out, since Windows tests are expensively slow. - # - 3.7 # - 3.8 - 3.9 - - "3.10.0-alpha - 3.10" + - "3.10" group: [1, 2] steps: diff --git a/docs/html/development/ci.rst b/docs/html/development/ci.rst index 39b9cdc8e74..bf78b53c71c 100644 --- a/docs/html/development/ci.rst +++ b/docs/html/development/ci.rst @@ -18,17 +18,17 @@ Supported interpreters pip support a variety of Python interpreters: -- CPython 3.6 - CPython 3.7 - CPython 3.8 - CPython 3.9 +- CPython 3.10 - Latest PyPy3 on different operating systems: - Linux - Windows -- MacOS +- macOS and on different architectures: @@ -77,9 +77,9 @@ Developer tasks ======== =============== ================ ================== ============= OS docs lint vendoring packaging ======== =============== ================ ================== ============= -Linux Github Github Github Github -Windows Github Github Github Github -MacOS Github Github Github Github +Linux GitHub GitHub GitHub GitHub +Windows GitHub GitHub GitHub GitHub +macOS GitHub GitHub GitHub GitHub ======== =============== ================ ================== ============= Actual testing @@ -88,28 +88,26 @@ Actual testing +------------------------------+---------------+-----------------+ | **interpreter** | **unit** | **integration** | +-----------+----------+-------+---------------+-----------------+ -| | | CP3.6 | | | -| | +-------+---------------+-----------------+ | | x86 | CP3.7 | | | | | +-------+---------------+-----------------+ | | | CP3.8 | | | | | +-------+---------------+-----------------+ | | | CP3.9 | | | | | +-------+---------------+-----------------+ +| | | CP3.10| | | +| | +-------+---------------+-----------------+ | | | PyPy3 | | | | Windows +----------+-------+---------------+-----------------+ -| | | CP3.6 | Github | Github | -| | +-------+---------------+-----------------+ -| | x64 | CP3.7 | | | +| | x64 | CP3.7 | GitHub | GitHub | | | +-------+---------------+-----------------+ | | | CP3.8 | | | | | +-------+---------------+-----------------+ -| | | CP3.9 | Github | Github | +| | | CP3.9 | GitHub | GitHub | +| | +-------+---------------+-----------------+ +| | | CP3.10| GitHub | GitHub | | | +-------+---------------+-----------------+ | | | PyPy3 | | | +-----------+----------+-------+---------------+-----------------+ -| | | CP3.6 | | | -| | +-------+---------------+-----------------+ | | x86 | CP3.7 | | | | | +-------+---------------+-----------------+ | | | CP3.8 | | | @@ -118,33 +116,33 @@ Actual testing | | +-------+---------------+-----------------+ | | | PyPy3 | | | | Linux +----------+-------+---------------+-----------------+ -| | | CP3.6 | Github | Github | +| | x64 | CP3.7 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | x64 | CP3.7 | Github | Github | +| | | CP3.8 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | | CP3.8 | Github | Github | +| | | CP3.9 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | | CP3.9 | Github | Github | +| | | CP3.10| GitHub | GitHub | | | +-------+---------------+-----------------+ | | | PyPy3 | | | +-----------+----------+-------+---------------+-----------------+ -| | | CP3.6 | | | -| | +-------+---------------+-----------------+ | | x86 | CP3.7 | | | | | +-------+---------------+-----------------+ | | | CP3.8 | | | | | +-------+---------------+-----------------+ | | | CP3.9 | | | | | +-------+---------------+-----------------+ +| | | CP3.10| | | +| | +-------+---------------+-----------------+ | | | PyPy3 | | | -| MacOS +----------+-------+---------------+-----------------+ -| | | CP3.6 | Github | Github | +| macOS +----------+-------+---------------+-----------------+ +| | x64 | CP3.7 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | x64 | CP3.7 | Github | Github | +| | | CP3.8 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | | CP3.8 | Github | Github | +| | | CP3.9 | GitHub | GitHub | | | +-------+---------------+-----------------+ -| | | CP3.9 | Github | Github | +| | | CP3.10| GitHub | GitHub | | | +-------+---------------+-----------------+ | | | PyPy3 | | | +-----------+----------+-------+---------------+-----------------+ diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst index 10dd271f6eb..3ba664f1231 100644 --- a/docs/html/development/getting-started.rst +++ b/docs/html/development/getting-started.rst @@ -70,15 +70,15 @@ To run tests: .. code-block:: console - $ tox -e py36 -- -n auto + $ tox -e py310 -- -n auto To run tests without parallelization, run: .. code-block:: console - $ tox -e py36 + $ tox -e py310 -The example above runs tests against Python 3.6. You can also use other +The example above runs tests against Python 3.10. You can also use other versions like ``py39`` and ``pypy3``. ``tox`` has been configured to forward any additional arguments it is given to @@ -88,11 +88,11 @@ can select tests using the various ways that pytest provides: .. code-block:: console $ # Using file name - $ tox -e py36 -- tests/functional/test_install.py + $ tox -e py310 -- tests/functional/test_install.py $ # Using markers - $ tox -e py36 -- -m unit + $ tox -e py310 -- -m unit $ # Using keywords - $ tox -e py36 -- -k "install and not wheel" + $ tox -e py310 -- -k "install and not wheel" Running pip's entire test suite requires supported version control tools (subversion, bazaar, git, and mercurial) to be installed. If you are missing @@ -101,8 +101,8 @@ explicitly tell pytest to skip those tests: .. code-block:: console - $ tox -e py36 -- -k "not svn" - $ tox -e py36 -- -k "not (svn or git)" + $ tox -e py310 -- -k "not svn" + $ tox -e py310 -- -k "not (svn or git)" Running Linters diff --git a/docs/html/installation.md b/docs/html/installation.md index 293750ff6ed..e5ee91831e9 100644 --- a/docs/html/installation.md +++ b/docs/html/installation.md @@ -75,7 +75,7 @@ $ pip install --upgrade pip The current version of pip works on: - Windows, Linux and MacOS. -- CPython 3.6, 3.7, 3.8, 3.9, 3.10 and latest PyPy3. +- CPython 3.7, 3.8, 3.9, 3.10 and latest PyPy3. pip is tested to work on the latest patch version of the Python interpreter, for each of the minor versions listed above. Previous patch versions are diff --git a/news/10641.removal.rst b/news/10641.removal.rst new file mode 100644 index 00000000000..2c165214443 --- /dev/null +++ b/news/10641.removal.rst @@ -0,0 +1 @@ +Drop support for Python 3.6. diff --git a/noxfile.py b/noxfile.py index 5b5a66d5307..06ad8d5493e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -69,7 +69,7 @@ def should_update_common_wheels() -> bool: # completely to nox for all our automation. Contributors should prefer using # `tox -e ...` until this note is removed. # ----------------------------------------------------------------------------- -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "pypy3"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "pypy3"]) def test(session: nox.Session) -> None: # Get the common wheels. if should_update_common_wheels(): diff --git a/setup.py b/setup.py index 4e5ce301526..fbb6a48a69c 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ def get_version(rel_path: str) -> str: "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -81,5 +80,5 @@ def get_version(rel_path: str) -> str: ], }, zip_safe=False, - python_requires=">=3.6", + python_requires=">=3.7", ) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index a30b45826a9..d326dc8cdb1 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -197,14 +197,7 @@ def install_requirements( if not requirements: return with contextlib.ExitStack() as ctx: - # TODO: Remove this block when dropping 3.6 support. Python 3.6 - # lacks importlib.resources and pep517 has issues loading files in - # a zip, so we fallback to the "old" method by adding the current - # pip directory to the child process's sys.path. - if sys.version_info < (3, 7): - pip_runnable = os.path.dirname(pip_location) - else: - pip_runnable = ctx.enter_context(_create_standalone_pip()) + pip_runnable = ctx.enter_context(_create_standalone_pip()) self._install_requirements( pip_runnable, finder, diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 75d8dd465ba..adac51058ce 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -16,7 +16,6 @@ from pip._internal.network.session import PipSession from pip._internal.utils.compat import stdlib_pkgs from pip._internal.utils.misc import tabulate, write_output -from pip._internal.utils.parallel import map_multithread if TYPE_CHECKING: from pip._internal.metadata.base import DistributionVersion @@ -254,7 +253,7 @@ def latest_info( dist.latest_filetype = typ return dist - for dist in map_multithread(latest_info, packages): + for dist in map(latest_info, packages): if dist is not None: yield dist diff --git a/src/pip/_internal/utils/parallel.py b/src/pip/_internal/utils/parallel.py deleted file mode 100644 index e31857732bf..00000000000 --- a/src/pip/_internal/utils/parallel.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Convenient parallelization of higher order functions. - -This module provides two helper functions, with appropriate fallbacks on -Python 2 and on systems lacking support for synchronization mechanisms: - -- map_multiprocess -- map_multithread - -These helpers work like Python 3's map, with two differences: - -- They don't guarantee the order of processing of - the elements of the iterable. -- The underlying process/thread pools chop the iterable into - a number of chunks, so that for very long iterables using - a large value for chunksize can make the job complete much faster - than using the default value of 1. -""" - -__all__ = ["map_multiprocess", "map_multithread"] - -from contextlib import contextmanager -from multiprocessing import Pool as ProcessPool -from multiprocessing import pool -from multiprocessing.dummy import Pool as ThreadPool -from typing import Callable, Iterable, Iterator, TypeVar, Union - -from pip._vendor.requests.adapters import DEFAULT_POOLSIZE - -Pool = Union[pool.Pool, pool.ThreadPool] -S = TypeVar("S") -T = TypeVar("T") - -# On platforms without sem_open, multiprocessing[.dummy] Pool -# cannot be created. -try: - import multiprocessing.synchronize # noqa -except ImportError: - LACK_SEM_OPEN = True -else: - LACK_SEM_OPEN = False - -# Incredibly large timeout to work around bpo-8296 on Python 2. -TIMEOUT = 2000000 - - -@contextmanager -def closing(pool: Pool) -> Iterator[Pool]: - """Return a context manager making sure the pool closes properly.""" - try: - yield pool - finally: - # For Pool.imap*, close and join are needed - # for the returned iterator to begin yielding. - pool.close() - pool.join() - pool.terminate() - - -def _map_fallback( - func: Callable[[S], T], iterable: Iterable[S], chunksize: int = 1 -) -> Iterator[T]: - """Make an iterator applying func to each element in iterable. - - This function is the sequential fallback either on Python 2 - where Pool.imap* doesn't react to KeyboardInterrupt - or when sem_open is unavailable. - """ - return map(func, iterable) - - -def _map_multiprocess( - func: Callable[[S], T], iterable: Iterable[S], chunksize: int = 1 -) -> Iterator[T]: - """Chop iterable into chunks and submit them to a process pool. - - For very long iterables using a large value for chunksize can make - the job complete much faster than using the default value of 1. - - Return an unordered iterator of the results. - """ - with closing(ProcessPool()) as pool: - return pool.imap_unordered(func, iterable, chunksize) - - -def _map_multithread( - func: Callable[[S], T], iterable: Iterable[S], chunksize: int = 1 -) -> Iterator[T]: - """Chop iterable into chunks and submit them to a thread pool. - - For very long iterables using a large value for chunksize can make - the job complete much faster than using the default value of 1. - - Return an unordered iterator of the results. - """ - with closing(ThreadPool(DEFAULT_POOLSIZE)) as pool: - return pool.imap_unordered(func, iterable, chunksize) - - -if LACK_SEM_OPEN: - map_multiprocess = map_multithread = _map_fallback -else: - map_multiprocess = _map_multiprocess - map_multithread = _map_multithread diff --git a/tests/unit/test_utils_parallel.py b/tests/unit/test_utils_parallel.py deleted file mode 100644 index b6c3e1fbfaa..00000000000 --- a/tests/unit/test_utils_parallel.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Test multiprocessing/multithreading higher-order functions.""" - -from contextlib import contextmanager -from importlib import import_module -from math import factorial -from sys import modules -from typing import Any, Iterator - -import pytest - -DUNDER_IMPORT = "builtins.__import__" -FUNC, ITERABLE = factorial, range(42) -MAPS = "map_multiprocess", "map_multithread" -_import = __import__ - - -def unload_parallel() -> None: - try: - del modules["pip._internal.utils.parallel"] - except KeyError: - pass - - -@contextmanager -def tmp_import_parallel() -> Iterator[Any]: - unload_parallel() - try: - yield import_module("pip._internal.utils.parallel") - finally: - unload_parallel() - - -def lack_sem_open(name: str, *args: Any, **kwargs: Any) -> Any: - """Raise ImportError on import of multiprocessing.synchronize.""" - if name.endswith("synchronize"): - raise ImportError - return _import(name, *args, **kwargs) - - -def have_sem_open(name: str, *args: Any, **kwargs: Any) -> Any: - """Make sure multiprocessing.synchronize import is successful.""" - # We don't care about the return value - # since we don't use the pool with this import. - if name.endswith("synchronize"): - return None - return _import(name, *args, **kwargs) - - -@pytest.mark.parametrize("name", MAPS) -def test_lack_sem_open(name: str, monkeypatch: pytest.MonkeyPatch) -> None: - """Test fallback when sem_open is not available. - - If so, multiprocessing[.dummy].Pool will fail to be created and - map_async should fallback to map. - """ - monkeypatch.setattr(DUNDER_IMPORT, lack_sem_open) - with tmp_import_parallel() as parallel: - assert getattr(parallel, name) is parallel._map_fallback - - -@pytest.mark.parametrize("name", MAPS) -def test_have_sem_open(name: str, monkeypatch: pytest.MonkeyPatch) -> None: - """Test fallback when sem_open is available.""" - monkeypatch.setattr(DUNDER_IMPORT, have_sem_open) - with tmp_import_parallel() as parallel: - assert getattr(parallel, name) is getattr(parallel, f"_{name}") - - -@pytest.mark.parametrize("name", MAPS) -def test_map(name: str) -> None: - """Test correctness of result of asynchronous maps.""" - map_async = getattr(import_module("pip._internal.utils.parallel"), name) - assert set(map_async(FUNC, ITERABLE)) == set(map(FUNC, ITERABLE)) diff --git a/tox.ini b/tox.ini index 9063c3ac340..b704e34382f 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ minversion = 3.4.0 envlist = docs, packaging, lint, vendoring, - py36, py37, py38, py39, py310, pypy3 + py37, py38, py39, py310, pypy3 [helpers] # Wrapper for calls to pip that make sure the version being used is the From 7c9da5f512076f0e80d6c7106b2d49adc8a71691 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 7 Nov 2021 13:47:15 +0200 Subject: [PATCH 2/2] Remove redundant Python 2 code --- setup.cfg | 2 -- tox.ini | 4 ---- 2 files changed, 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index f8f069ef722..2a1c574c8c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -107,8 +107,6 @@ exclude_lines = pragma: no cover # This excludes typing-specific code, which will be validated by mypy anyway. if TYPE_CHECKING - # Can be set to exclude e.g. `if PY2:` on Python 3 - ${PIP_CI_COVERAGE_EXCLUDES} [metadata] license_file = LICENSE.txt diff --git a/tox.ini b/tox.ini index b704e34382f..23208736df8 100644 --- a/tox.ini +++ b/tox.ini @@ -11,11 +11,9 @@ pip = python {toxinidir}/tools/tox_pip.py mkdirp = python -c 'import os, sys; os.path.exists(sys.argv[1]) or os.mkdir(sys.argv[1])' [testenv] -# Remove USERNAME once we drop PY2. passenv = CI GIT_SSL_CAINFO - USERNAME HTTP_PROXY HTTPS_PROXY NO_PROXY @@ -43,8 +41,6 @@ setenv = # Ensure coverage is enabled in child processes in virtual environments # since they won't already have been enabled by pytest-cov. COVERAGE_PROCESS_START = {toxinidir}/setup.cfg - # Used in coverage configuration in setup.cfg. - PIP_CI_COVERAGE_EXCLUDES = if PY2 [testenv:docs] # Don't skip install here since pip_sphinxext uses pip's internals.