diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml deleted file mode 100644 index 17d841f1f1c54ac..000000000000000 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Verify bundled wheels - -on: - workflow_dispatch: - push: - paths: - - 'Lib/ensurepip/_bundled/**' - - '.github/workflows/verify-ensurepip-wheels.yml' - - 'Tools/build/verify_ensurepip_wheels.py' - pull_request: - paths: - - 'Lib/ensurepip/_bundled/**' - - '.github/workflows/verify-ensurepip-wheels.yml' - - 'Tools/build/verify_ensurepip_wheels.py' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - verify: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3' - - name: Compare checksum of bundled wheels to the ones published on PyPI - run: ./Tools/build/verify_ensurepip_wheels.py diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 1fb1d505cfd0c53..78c216d95ca1b9a 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,16 @@ -import collections +"""Bundled Pip installer.""" + import os -import os.path +import pathlib +import shutil import subprocess import sys -import sysconfig import tempfile -from importlib import resources - -__all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.2.1" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] +from ._wheelhouses import discover_ondisk_packages -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) -# Directory of system wheel packages. Some Linux distribution packaging -# policies recommend against bundling dependencies. For example, Fedora -# installs wheel packages in the /usr/share/python-wheels/ directory and don't -# install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') - - -def _find_packages(path): - packages = {} - try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None +__all__ = ("version", "bootstrap") def _run_pip(args, additional_paths=None): @@ -105,7 +43,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return discover_ondisk_packages()['pip'].project_version def _disable_pip_configuration_settings(): @@ -164,27 +102,18 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # omit pip os.environ["ENSUREPIP_OPTIONS"] = "install" + ondisk_dist_pkgs_map = discover_ondisk_packages() with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path + tmpdir_path = pathlib.Path(tmpdir) additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + for package in ondisk_dist_pkgs_map.values(): + with package.as_pathlib_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + shutil.copy2(bundled_wheel_path, tmp_wheel_path) + + additional_paths.append(str(tmp_wheel_path)) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +126,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + bundled_project_names = list(ondisk_dist_pkgs_map.keys()) + return _run_pip(args + bundled_project_names, additional_paths) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +158,8 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + bundled_project_names = list(discover_ondisk_packages().keys()) + return _run_pip(args + bundled_project_names) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/.gitignore b/Lib/ensurepip/_bundled/.gitignore new file mode 100644 index 000000000000000..7c9d611b59248fe --- /dev/null +++ b/Lib/ensurepip/_bundled/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!README.md diff --git a/Lib/ensurepip/_bundled/README.md b/Lib/ensurepip/_bundled/README.md new file mode 100644 index 000000000000000..98c358d4a92c4e0 --- /dev/null +++ b/Lib/ensurepip/_bundled/README.md @@ -0,0 +1,23 @@ +# Upstream packaging + +To populate this directory, the initial build packagers are supposed +to invoke the following command: + +```console +$ python -m ensurepip.bundle +``` + +It will download a pre-defined version of the Pip wheel. Its SHA-256 +hash is guaranteed to match the one on PyPI. + +# Downstream packaging + +Packagers of the downstream distributions are welcome to put an +alternative wheel version in the directory defined by the +`WHEEL_PKG_DIR` configuration setting. If this is done, + +```console +$ python -m ensurepip +``` + +will prefer the replacement distribution package over the bundled one. diff --git a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl deleted file mode 100644 index ba28ef02e265f03..000000000000000 Binary files a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundler.py b/Lib/ensurepip/_bundler.py new file mode 100644 index 000000000000000..8df324b9b9bb21c --- /dev/null +++ b/Lib/ensurepip/_bundler.py @@ -0,0 +1,40 @@ +"""Build time dist downloading and bundling logic.""" + +from __future__ import annotations + +import sys +from contextlib import suppress +from importlib.resources import as_file as _traversable_to_pathlib_ctx + +from ._structs import BUNDLED_WHEELS_PATH, REMOTE_DIST_PKGS + + +def ensure_wheels_are_downloaded(*, verbosity: bool = False) -> None: + """Download wheels into bundle if they are not there yet.""" + for pkg in REMOTE_DIST_PKGS: + existing_whl_file_path = BUNDLED_WHEELS_PATH / pkg.wheel_file_name + with suppress(FileNotFoundError): + if pkg.matches(existing_whl_file_path.read_bytes()): + if verbosity: + print( + f'A valid `{pkg.wheel_file_name}` is already ' + 'present in cache. Skipping download.', + file=sys.stderr, + ) + continue + + if verbosity: + print( + f'Downloading `{pkg.wheel_file_name}`...', + file=sys.stderr, + ) + downloaded_whl_contents = pkg.download_verified_wheel_contents() + + if verbosity: + print( + f'Saving `{pkg.wheel_file_name}` to disk...', + file=sys.stderr, + ) + with _traversable_to_pathlib_ctx(BUNDLED_WHEELS_PATH) as bundled_dir: + whl_file_path = bundled_dir / pkg.wheel_file_name + whl_file_path.write_bytes(downloaded_whl_contents) diff --git a/Lib/ensurepip/_structs.py b/Lib/ensurepip/_structs.py new file mode 100644 index 000000000000000..f8514a0057f4328 --- /dev/null +++ b/Lib/ensurepip/_structs.py @@ -0,0 +1,251 @@ +"""Data structures to make the control flow easy.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager, nullcontext +from hashlib import sha256 as _compute_sha256 +from dataclasses import dataclass +from functools import cache, cached_property +from importlib.resources import ( + abc as _resources_abc, + as_file as _traversable_to_pathlib_ctx, + files as _get_traversable_dir_for, +) +from pathlib import Path +from sysconfig import get_config_var +from urllib.request import urlopen + + +BUNDLED_WHEELS_PATH: _resources_abc.Traversable = ( + _get_traversable_dir_for(__package__) / "_bundled" +) +"""Path to the wheels that CPython bundles within ensurepip.""" + +_WHL_TAG: str = 'py3' + + +@cache +def _convert_project_spec_to_wheel_name( + project_name: str, + project_version: str, +) -> str: + return ( + f'{project_name !s}-{project_version !s}-' + f'{_WHL_TAG !s}-none-any.whl' + ) + + +@cache +def _get_wheel_pkg_dir_from_sysconfig() -> Path: + """Read path to wheels Linux downstream distributions prefer.""" + # `WHEEL_PKG_DIR` is a directory of system wheel packages. Some Linux + # distribution packaging policies recommend against bundling dependencies. + # For example, Fedora installs wheel packages in the + # `/usr/share/python-wheels/` directory and don't install + # the `ensurepip._bundled` package. + try: + return Path(get_config_var('WHEEL_PKG_DIR')).resolve() + except TypeError as none_type_err: + raise LookupError( + 'The compile-time `WHEEL_PKG_DIR` is unset so there is ' + 'no place for looking up the wheels.', + ) from none_type_err + + +@dataclass(frozen=True) +class _RemoteDistributionPackage: + """Structure representing a wheel on PyPI.""" + + project_name: str + """PyPI project name.""" + + project_version: str + """PyPI project version.""" + + wheel_sha256: str + """PyPI wheel SHA-256 hash.""" + + @cached_property + def wheel_file_name(self) -> str: + """Name of the wheel file on remote server.""" + return _convert_project_spec_to_wheel_name( + self.project_name, self.project_version, + ) + + @cached_property + def wheel_file_url(self) -> str: + """URL to the wheel file on remote server.""" + return ( + f'https://files.pythonhosted.org/packages/{_WHL_TAG !s}/' + f'{self.project_name[0] !s}/{self.project_name !s}/' + f'{self.wheel_file_name !s}' + ) + + @cached_property + def verifiable_wheel_file_url(self) -> str: + """URL to the wheel file on remote server that includes hash.""" + return f'{self.wheel_file_url !s}#sha256={self.wheel_sha256 !s}' + + def download_verified_wheel_contents(self) -> memoryview: + """Retrieve the remote wheel contents and verify its hash. + + :raises ValueError: If the recorded SHA-256 hash doesn't match + the downloaded payload. + + Returns the URL contents as a :py:class:`memoryview` object + on success. + """ + with urlopen(self.wheel_file_url) as downloaded_fd: + resource_content = memoryview(downloaded_fd.read()) + + if not self.matches(resource_content): + raise ValueError(f"The payload's hash is invalid for {self !r}.") + + return resource_content + + def matches( + self, + wheel_content: bytes | memoryview, + /, + ) -> bool: + """Verify the content's SHA-256 hash against recorded value.""" + return self.wheel_sha256 == _compute_sha256(wheel_content).hexdigest() + + def __repr__(self) -> str: + """Render remote distribution package instance for humans.""" + return ( + f'{self.project_name !s} == {self.project_version !s}' + f' @ {self.verifiable_wheel_file_url !s}' + ) + + +@dataclass(frozen=True) +class BundledDistributionPackage: + """Structure representing a wheel under ``ensurepip/_bundled/``.""" + + project_name: str + """PyPI project name.""" + + project_version: str + """PyPI project version.""" + + wheel_name: str + """Wheel package file name.""" + + wheel_path: _resources_abc.Traversable + """Wheel package file path.""" + + @classmethod + def from_project_spec( + cls, + project_name: str, + project_version: str, + /, + ) -> BundledDistributionPackage: + """Create a replacement package from name and version spec.""" + wheel_name = _convert_project_spec_to_wheel_name( + project_name, project_version, + ) + wheel_path = BUNDLED_WHEELS_PATH / wheel_name + return cls(project_name, project_version, wheel_name, wheel_path) + + @classmethod + def from_remote_dist( + cls, + remote_distribution_pkg: _RemoteDistributionPackage, + /, + ) -> BundledDistributionPackage: + """Create a replacement package from its remote counterpart.""" + return cls.from_project_spec( + remote_distribution_pkg.project_name, + remote_distribution_pkg.project_version, + ) + + def as_pathlib_ctx(self) -> AbstractContextManager[Path]: + """Make a context manager exposing a :mod:`pathlib` instance.""" + return _traversable_to_pathlib_ctx(self.wheel_path) + + +@dataclass(frozen=True) +class ReplacementDistributionPackage: + """Structure representing a wheel under ``WHEEL_PKG_DIR``.""" + + project_name: str + """PyPI project name.""" + + project_version: str + """PyPI project version.""" + + wheel_name: str + """Wheel package file name.""" + + wheel_path: Path + """Wheel package file path.""" + + @classmethod + def from_distribution_package_name( + cls, + dist_pkg_name: str, + /, + ) -> ReplacementDistributionPackage: + """Look up a replacement package from name. + + :raises LookupError: If ``WHEEL_PKG_DIR`` is not set or the package + wheel is nowhere to be found. + + """ + wheel_file_name_prefix = f'{dist_pkg_name !s}-' + + wheel_pkg_dir = _get_wheel_pkg_dir_from_sysconfig() + dist_matching_wheels = wheel_pkg_dir.glob( + f'{wheel_file_name_prefix !s}*.whl', + ) + + try: + first_matching_dist_wheel = sorted(dist_matching_wheels)[0] + except IndexError as index_err: + raise LookupError( + '`WHEEL_PKG_DIR` does not contain any wheel files ' + f'for `{dist_pkg_name !s}`.', + ) from index_err + + wheel_name = first_matching_dist_wheel.name + dist_pkg_version = ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix(wheel_file_name_prefix). + partition('-')[0] + ) + + return cls( + dist_pkg_name, dist_pkg_version, + wheel_name, first_matching_dist_wheel, + ) + + @classmethod + def from_remote_dist( + cls, + remote_distribution_pkg: _RemoteDistributionPackage, + /, + ) -> ReplacementDistributionPackage: + """Create a replacement package from its remote counterpart.""" + return cls.from_distribution_package_name( + remote_distribution_pkg.project_name, + ) + + def as_pathlib_ctx(self) -> AbstractContextManager[Path]: + """Make a context manager exposing a :mod:`pathlib` instance.""" + return nullcontext(self.wheel_path) + + +PIP_REMOTE_DIST = _RemoteDistributionPackage( + 'pip', + '23.2.1', + '7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be', +) +"""Pip distribution package on PyPI.""" + +REMOTE_DIST_PKGS = ( + PIP_REMOTE_DIST, +) +"""Distribution packages provisioned by ``ensurepip``.""" diff --git a/Lib/ensurepip/_wheelhouses.py b/Lib/ensurepip/_wheelhouses.py new file mode 100644 index 000000000000000..db10ea1fa65b92b --- /dev/null +++ b/Lib/ensurepip/_wheelhouses.py @@ -0,0 +1,51 @@ +"""Pre-bundled wheel discovery.""" + +from __future__ import annotations + +from functools import cache + +from ._structs import ( + BundledDistributionPackage, + ReplacementDistributionPackage, + REMOTE_DIST_PKGS, +) + + +def _discover_ondisk_bundled_packages() -> tuple[ + BundledDistributionPackage, + ..., +]: + return tuple( + BundledDistributionPackage.from_remote_dist(remote_pkg) + for remote_pkg in REMOTE_DIST_PKGS + ) + + +def _discover_ondisk_replacement_packages() -> tuple[ + ReplacementDistributionPackage, + ..., +]: + try: + return tuple( + ReplacementDistributionPackage.from_remote_dist(remote_pkg) + for remote_pkg in REMOTE_DIST_PKGS + ) + except LookupError: + return () + + +@cache +def discover_ondisk_packages() -> dict[ + str, + BundledDistributionPackage | ReplacementDistributionPackage, +]: + """Return a mapping of packages found on disk. + + The result is either a list of distribution packages from + ``WHEEL_PKG_DIR`` xor the bundled one. + """ + ondisk_packages = ( + _discover_ondisk_replacement_packages() + or _discover_ondisk_bundled_packages() + ) + return {pkg.project_name: pkg for pkg in ondisk_packages} diff --git a/Lib/ensurepip/bundle.py b/Lib/ensurepip/bundle.py new file mode 100644 index 000000000000000..803b02afba74119 --- /dev/null +++ b/Lib/ensurepip/bundle.py @@ -0,0 +1,9 @@ +"""Download Pip and setuptools dists for bundling.""" + +import sys + +from ._bundler import ensure_wheels_are_downloaded + + +if __name__ == '__main__': + ensure_wheels_are_downloaded(verbosity='-v' in sys.argv) diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 69ab2a4feaa9389..2b6bc71aacb707c 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,12 +6,29 @@ import test.support import unittest import unittest.mock +import urllib.request +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from random import randbytes, randint +from runpy import run_module import ensurepip +import ensurepip._bundler +import ensurepip._structs +import ensurepip._wheelhouses import ensurepip._uninstall class TestPackages(unittest.TestCase): + def setUp(self): + ensurepip._structs._get_wheel_pkg_dir_from_sysconfig.cache_clear() + ensurepip._wheelhouses.discover_ondisk_packages.cache_clear() + + def tearDown(self): + ensurepip._structs._get_wheel_pkg_dir_from_sysconfig.cache_clear() + ensurepip._wheelhouses.discover_ondisk_packages.cache_clear() + def touch(self, directory, filename): fullname = os.path.join(directory, filename) open(fullname, "wb").close() @@ -20,41 +37,63 @@ def test_version(self): # Test version() with tempfile.TemporaryDirectory() as tmpdir: self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + with unittest.mock.patch.object( + ensurepip._structs, 'get_config_var', + lambda name: Path(tmpdir), + ): self.assertEqual(ensurepip.version(), '1.2.3b1') def test_get_packages_no_dir(self): # Test _get_packages() without a wheel package directory - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): - packages = ensurepip._get_packages() - - # when bundled wheel packages are used, we get _PIP_VERSION - self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + with unittest.mock.patch.object( + ensurepip._structs, 'get_config_var', + lambda name: None, + ): + # when bundled wheel packages are used, we get bundled pip version + self.assertEqual( + ensurepip._structs.PIP_REMOTE_DIST.project_version, + ensurepip.version(), + ) # use bundled wheel packages - self.assertIsNotNone(packages['pip'].wheel_name) + self.assertTrue( + isinstance( + ensurepip._wheelhouses.discover_ondisk_packages()['pip'], + ensurepip._structs.BundledDistributionPackage, + ), + ) def test_get_packages_with_dir(self): # Test _get_packages() with a wheel package directory pip_filename = "pip-20.2.2-py2.py3-none-any.whl" with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) self.touch(tmpdir, pip_filename) # not used, make sure that it's ignored self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") - with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), - unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): - packages = ensurepip._get_packages() + with unittest.mock.patch.object( + ensurepip._structs, 'get_config_var', + lambda name: tmp_path, + ): + packages = ensurepip._wheelhouses.discover_ondisk_packages() - self.assertEqual(packages['pip'].version, '20.2.2') - self.assertEqual(packages['pip'].wheel_path, - os.path.join(tmpdir, pip_filename)) + self.assertEqual(packages['pip'].project_version, '20.2.2') + self.assertEqual( + packages['pip'].wheel_path, + tmp_path / pip_filename, + ) # wheel package is ignored - self.assertEqual(sorted(packages), ['pip']) + self.assertEqual(sorted(packages.keys()), ['pip']) + + def test_returns_version(self): + pip_url = ensurepip._structs.PIP_REMOTE_DIST.wheel_file_url + self.assertIn( + f'/packages/py3/p/pip/pip-{ensurepip.version()}-', + pip_url, + ) class EnsurepipMixin: @@ -69,13 +108,14 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() - # But expose os.listdir() used by _find_packages() - patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull - patched_os.path = os.path self.os_environ = patched_os.environ = os.environ.copy() + shutil_patch = unittest.mock.patch("ensurepip.shutil") + shutil_patch.start() + self.addCleanup(shutil_patch.stop) + class TestBootstrap(EnsurepipMixin, unittest.TestCase): @@ -93,6 +133,34 @@ def test_basic_bootstrapping(self): additional_paths = self.run_pip.call_args[0][1] self.assertEqual(len(additional_paths), 1) + def test_replacement_wheel_bootstrapping(self): + ensurepip._structs._get_wheel_pkg_dir_from_sysconfig.cache_clear() + ensurepip._wheelhouses.discover_ondisk_packages.cache_clear() + + pip_wheel_name = ( + f'pip-{ensurepip._structs.PIP_REMOTE_DIST.project_version !s}-' + 'py3-none-any.whl' + ) + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + tmp_wheel_path = tmp_path / pip_wheel_name + tmp_wheel_path.touch() + + with unittest.mock.patch.object( + ensurepip._structs, 'get_config_var', + lambda name: tmp_path, + ): + ensurepip.bootstrap() + + ensurepip._structs._get_wheel_pkg_dir_from_sysconfig.cache_clear() + ensurepip._wheelhouses.discover_ondisk_packages.cache_clear() + + additional_paths = self.run_pip.call_args[0][1] + self.assertTrue( + additional_paths[-1].endswith(f'{os.sep !s}{pip_wheel_name !s}'), + ) + def test_bootstrapping_with_root(self): ensurepip.bootstrap(root="/foo/bar/") @@ -190,6 +258,159 @@ def test_pip_config_file_disabled(self): ensurepip.bootstrap() self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) + +class TestBundle(EnsurepipMixin, unittest.TestCase): + def test_wheel_hash_mismatch(self): + wheel_contents_stub = memoryview(randbytes(randint(256, 512))) + sha256_hash = sha256(wheel_contents_stub).hexdigest() + remote_dist_stub = ensurepip._structs._RemoteDistributionPackage( + 'pypi-pkg', '3.2.1', + sha256_hash, + ) + + class MockedHTTPSOpener: + def open(self, url, data, timeout): + assert 'pypi-pkg' in url + assert data is None # HTTP GET + # Intentionally corrupt the wheel: + return BytesIO(wheel_contents_stub.tobytes()[:-1]) + + with ( + unittest.mock.patch.object( + urllib.request, + '_opener', + None, + ), + self.assertRaisesRegex( + ValueError, + r"^The payload's hash is invalid for ", + ) + ): + urllib.request.install_opener(MockedHTTPSOpener()) + remote_dist_stub.download_verified_wheel_contents() + + def test_bundle_cached(self): + wheel_contents_stub = memoryview(randbytes(randint(256, 512))) + sha256_hash = sha256(wheel_contents_stub).hexdigest() + remote_dist_stub = ensurepip._structs._RemoteDistributionPackage( + 'pip', '1.2.3', + sha256_hash, + ) + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + ( + tmp_path / remote_dist_stub.wheel_file_name + ).write_bytes(wheel_contents_stub) + test_cases = ( + ('no CLI args', (), []), + ( + 'verbose', + ('-v',), + [ + unittest.mock.call( + 'A valid `pip-1.2.3-py3-none-any.whl` ' + 'is already present in cache. Skipping download.', + ), + unittest.mock.call('\n') + ] + ), + ) + for case_name, case_cli_args, expected_stderr_writes in test_cases: + with self.subTest(case_name): + with ( + unittest.mock.patch.object( + ensurepip._bundler, 'BUNDLED_WHEELS_PATH', + tmp_path, + ), + unittest.mock.patch.object( + ensurepip._bundler, + 'REMOTE_DIST_PKGS', + [remote_dist_stub], + ), + unittest.mock.patch.object( + sys, 'argv', [sys.executable, *case_cli_args], + ), + unittest.mock.patch.object( + sys.stderr, 'write', + ) as stderr_write_mock, + ): + run_module('ensurepip.bundle', run_name='__main__') + self.assertEqual( + stderr_write_mock.call_args_list, + expected_stderr_writes, + ) + + def test_bundle_download(self): + wheel_contents_stub = memoryview(randbytes(randint(256, 512))) + sha256_hash = sha256(wheel_contents_stub).hexdigest() + remote_dist_stub = ensurepip._structs._RemoteDistributionPackage( + 'pip', '1.2.3', + sha256_hash, + ) + + class MockedHTTPSOpener: + def open(self, url, data, timeout): + assert 'pip' in url + assert data is None # HTTP GET + return BytesIO(wheel_contents_stub) + + test_cases = ( + ('no CLI args', (), []), + ( + 'verbose', + ('-v',), + [ + unittest.mock.call( + 'Downloading `pip-1.2.3-py3-none-any.whl`...' + ), + unittest.mock.call('\n'), + unittest.mock.call( + 'Saving `pip-1.2.3-py3-none-any.whl` to disk...', + ), + unittest.mock.call('\n') + ] + ), + ) + for case_name, case_cli_args, expected_stderr_writes in test_cases: + with self.subTest(case_name): + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + with ( + unittest.mock.patch.object( + ensurepip._bundler, 'BUNDLED_WHEELS_PATH', + Path(tmpdir), + ), + unittest.mock.patch.object( + ensurepip._bundler, + 'REMOTE_DIST_PKGS', + [remote_dist_stub], + ), + unittest.mock.patch.object( + sys, 'argv', [sys.executable, *case_cli_args], + ), + unittest.mock.patch.object( + sys.stderr, 'write', + ) as stderr_write_mock, + unittest.mock.patch.object( + urllib.request, + '_opener', + None, + ), + ): + urllib.request.install_opener(MockedHTTPSOpener()) + run_module('ensurepip.bundle', run_name='__main__') + self.assertEqual( + ( + tmp_path / remote_dist_stub.wheel_file_name + ).read_bytes(), + wheel_contents_stub, + ) + self.assertEqual( + stderr_write_mock.call_args_list, + expected_stderr_writes, + ) + + @contextlib.contextmanager def fake_pip(version=ensurepip.version()): if version is None: diff --git a/Misc/NEWS.d/next/Library/2019-04-11-22-21-28.bpo-36608.HFoazc.rst b/Misc/NEWS.d/next/Library/2019-04-11-22-21-28.bpo-36608.HFoazc.rst new file mode 100644 index 000000000000000..c27317afc605e6f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-04-11-22-21-28.bpo-36608.HFoazc.rst @@ -0,0 +1 @@ +Replaced vendoring of pip blob in the CPython Git tree with a build-time download and caching mechanism. diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py deleted file mode 100755 index 09fd5d9e3103ac9..000000000000000 --- a/Tools/build/verify_ensurepip_wheels.py +++ /dev/null @@ -1,98 +0,0 @@ -#! /usr/bin/env python3 - -""" -Compare checksums for wheels in :mod:`ensurepip` against the Cheeseshop. - -When GitHub Actions executes the script, output is formatted accordingly. -https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message -""" - -import hashlib -import json -import os -import re -from pathlib import Path -from urllib.request import urlopen - -PACKAGE_NAMES = ("pip",) -ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip" -WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled" -ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8") -GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" - - -def print_notice(file_path: str, message: str) -> None: - if GITHUB_ACTIONS: - message = f"::notice file={file_path}::{message}" - print(message, end="\n\n") - - -def print_error(file_path: str, message: str) -> None: - if GITHUB_ACTIONS: - message = f"::error file={file_path}::{message}" - print(message, end="\n\n") - - -def verify_wheel(package_name: str) -> bool: - # Find the package on disk - package_path = next(WHEEL_DIR.glob(f"{package_name}*.whl"), None) - if not package_path: - print_error("", f"Could not find a {package_name} wheel on disk.") - return False - - print(f"Verifying checksum for {package_path}.") - - # Find the version of the package used by ensurepip - package_version_match = re.search( - f'_{package_name.upper()}_VERSION = "([^"]+)', ENSURE_PIP_INIT_PY_TEXT - ) - if not package_version_match: - print_error( - package_path, - f"No {package_name} version found in Lib/ensurepip/__init__.py.", - ) - return False - package_version = package_version_match[1] - - # Get the SHA 256 digest from the Cheeseshop - try: - raw_text = urlopen(f"https://pypi.org/pypi/{package_name}/json").read() - except (OSError, ValueError): - print_error(package_path, f"Could not fetch JSON metadata for {package_name}.") - return False - - release_files = json.loads(raw_text)["releases"][package_version] - for release_info in release_files: - if package_path.name != release_info["filename"]: - continue - expected_digest = release_info["digests"].get("sha256", "") - break - else: - print_error(package_path, f"No digest for {package_name} found from PyPI.") - return False - - # Compute the SHA 256 digest of the wheel on disk - actual_digest = hashlib.sha256(package_path.read_bytes()).hexdigest() - - print(f"Expected digest: {expected_digest}") - print(f"Actual digest: {actual_digest}") - - if actual_digest != expected_digest: - print_error( - package_path, f"Failed to verify the checksum of the {package_name} wheel." - ) - return False - - print_notice( - package_path, - f"Successfully verified the checksum of the {package_name} wheel.", - ) - return True - - -if __name__ == "__main__": - exit_status = 0 - for package_name in PACKAGE_NAMES: - if not verify_wheel(package_name): - exit_status = 1 - raise SystemExit(exit_status)