From a984fdf1c8e50bb1c7b565d19eaab13417f36add Mon Sep 17 00:00:00 2001 From: Vincent Philippon Date: Thu, 28 Sep 2017 17:29:29 -0400 Subject: [PATCH 1/2] Keep pip.Wheel monkey patch only in the get_hashes context. Fixes #558 --- CHANGELOG.md | 5 +++ piptools/repositories/pypi.py | 62 +++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 015ea4096..d4198ee5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 1.10.2 (UNRELEASED) + +Bug Fixes: +- Fixed bug causing dependencies from invalid wheels for the current platform to be included ([#571](https://github.com/jazzband/pip-tools/pull/571)). + # 1.10.1 (2017-09-27) Bug Fixes: diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 598fc9724..e10f2311f 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -28,24 +28,6 @@ from .._compat import TemporaryDirectory -# Monkey patch pip's Wheel class to support all platform tags. This allows -# pip-tools to generate hashes for all available distributions, not only the -# one for the current platform. - -def _wheel_supported(self, tags=None): - # Ignore current platform. Support everything. - return True - - -def _wheel_support_index_min(self, tags=None): - # All wheels are equal priority for sorting. - return 0 - - -Wheel.supported = _wheel_supported -Wheel.support_index_min = _wheel_support_index_min - - class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = 'https://pypi.python.org/simple' @@ -182,12 +164,13 @@ def get_hashes(self, ireq): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly - # satisify this constraint. - all_candidates = self.find_all_candidates(ireq.name) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - matching_versions = list( - ireq.specifier.filter((candidate.version for candidate in all_candidates))) - matching_candidates = candidates_by_version[matching_versions[0]] + # satisfy this constraint. + with self.allow_all_wheels(): + all_candidates = self.find_all_candidates(ireq.name) + candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) + matching_versions = list( + ireq.specifier.filter((candidate.version for candidate in all_candidates))) + matching_candidates = candidates_by_version[matching_versions[0]] return { self._get_file_hash(candidate.location) @@ -201,6 +184,37 @@ def _get_file_hash(self, location): h.update(chunk) return ":".join([FAVORITE_HASH, h.hexdigest()]) + @contextmanager + def allow_all_wheels(self): + """ + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. + + This also saves the candidate cache and set a new one, or else the results from the + previous non-patched calls will interfere. + """ + def _wheel_supported(self, tags=None): + # Ignore current platform. Support everything. + return True + + def _wheel_support_index_min(self, tags=None): + # All wheels are equal priority for sorting. + return 0 + + original_wheel_supported = Wheel.supported + original_support_index_min = Wheel.support_index_min + original_cache = self._available_candidates_cache + + Wheel.supported = _wheel_supported + Wheel.support_index_min = _wheel_support_index_min + self._available_candidates_cache = {} + + try: + yield + finally: + Wheel.supported = original_wheel_supported + Wheel.support_index_min = original_support_index_min + self._available_candidates_cache = original_cache + @contextmanager def open_local_or_remote_file(link, session): From 81cf817c87b2b64d047595bf4cc503383e6989a3 Mon Sep 17 00:00:00 2001 From: Vincent Philippon Date: Thu, 28 Sep 2017 18:42:34 -0400 Subject: [PATCH 2/2] Change usage to reuse the cache for the hash computation. --- piptools/repositories/base.py | 8 ++++++++ piptools/repositories/local.py | 7 +++++++ piptools/repositories/pypi.py | 11 +++++------ piptools/resolver.py | 3 ++- tests/conftest.py | 6 ++++++ tests/test_repository_pypi.py | 3 ++- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py index b791eab26..57e85fda0 100644 --- a/piptools/repositories/base.py +++ b/piptools/repositories/base.py @@ -3,6 +3,7 @@ unicode_literals) from abc import ABCMeta, abstractmethod +from contextlib import contextmanager from six import add_metaclass @@ -38,3 +39,10 @@ def get_hashes(self, ireq): all of the files for a given requirement. It is not acceptable for an editable or unpinned requirement to be passed to this function. """ + + @abstractmethod + @contextmanager + def allow_all_wheels(self): + """ + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. + """ diff --git a/piptools/repositories/local.py b/piptools/repositories/local.py index ea3a39b98..8b30052e9 100644 --- a/piptools/repositories/local.py +++ b/piptools/repositories/local.py @@ -2,6 +2,8 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +from contextlib import contextmanager + from piptools.utils import as_tuple, key_from_req, make_install_requirement from .base import BaseRepository @@ -63,3 +65,8 @@ def get_dependencies(self, ireq): def get_hashes(self, ireq): return self.repository.get_hashes(ireq) + + @contextmanager + def allow_all_wheels(self): + with self.repository.allow_all_wheels(): + yield diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index e10f2311f..26b6751ff 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -165,12 +165,11 @@ def get_hashes(self, ireq): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - with self.allow_all_wheels(): - all_candidates = self.find_all_candidates(ireq.name) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - matching_versions = list( - ireq.specifier.filter((candidate.version for candidate in all_candidates))) - matching_candidates = candidates_by_version[matching_versions[0]] + all_candidates = self.find_all_candidates(ireq.name) + candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) + matching_versions = list( + ireq.specifier.filter((candidate.version for candidate in all_candidates))) + matching_candidates = candidates_by_version[matching_versions[0]] return { self._get_file_hash(candidate.location) diff --git a/piptools/resolver.py b/piptools/resolver.py index 2906265ca..8c4e98114 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -68,7 +68,8 @@ def resolve_hashes(self, ireqs): """ Finds acceptable hashes for all of the given InstallRequirements. """ - return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} + with self.repository.allow_all_wheels(): + return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} def resolve(self, max_rounds=10): """ diff --git a/tests/conftest.py b/tests/conftest.py index 62de67f94..e8eda9394 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import json +from contextlib import contextmanager from functools import partial from pip._vendor.packaging.version import Version @@ -49,6 +50,11 @@ def get_dependencies(self, ireq): dependencies = [dep for extra in extras for dep in self.index[name][version][extra]] return [InstallRequirement.from_line(dep, constraint=ireq.constraint) for dep in dependencies] + @contextmanager + def allow_all_wheels(self): + # No need to do an actual pip.Wheel mock here. + yield + class FakeInstalledDistribution(object): def __init__(self, line, deps=None): diff --git a/tests/test_repository_pypi.py b/tests/test_repository_pypi.py index cea0479b9..97c4156de 100644 --- a/tests/test_repository_pypi.py +++ b/tests/test_repository_pypi.py @@ -52,7 +52,8 @@ def test_generate_hashes_all_platforms(from_line): session = pip_command._build_session(pip_options) repository = PyPIRepository(pip_options, session) ireq = from_line('cffi==1.9.1') - assert repository.get_hashes(ireq) == expected + with repository.allow_all_wheels(): + assert repository.get_hashes(ireq) == expected def test_generate_hashes_without_interfering_with_each_other(from_line):