From a012fbe2bf1379e01f103c74352d33dc590de200 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 3 Sep 2020 17:07:51 +0800 Subject: [PATCH] Pull in hashes from constraint files --- news/8792.bugfix | 2 + .../_internal/resolution/resolvelib/base.py | 37 +++++++++++++++++- .../resolution/resolvelib/factory.py | 11 ++++-- .../resolution/resolvelib/provider.py | 7 ++-- .../resolution/resolvelib/resolver.py | 12 +++--- tests/functional/test_new_resolver_hashes.py | 38 +++++++++++++++++++ .../resolution_resolvelib/test_requirement.py | 7 ++-- 7 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 news/8792.bugfix diff --git a/news/8792.bugfix b/news/8792.bugfix new file mode 100644 index 00000000000..e83bdb09cfe --- /dev/null +++ b/news/8792.bugfix @@ -0,0 +1,2 @@ +New resolver: Pick up hash declarations in constraints files and use them to +filter available distributions. diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py index 9245747bf2b..7c09cd70b8d 100644 --- a/src/pip/_internal/resolution/resolvelib/base.py +++ b/src/pip/_internal/resolution/resolvelib/base.py @@ -1,5 +1,8 @@ +from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.hashes import Hashes from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -8,7 +11,6 @@ from pip._vendor.packaging.version import _BaseVersion from pip._internal.models.link import Link - from pip._internal.req.req_install import InstallRequirement CandidateLookup = Tuple[ Optional["Candidate"], @@ -24,6 +26,39 @@ def format_name(project, extras): return "{}[{}]".format(project, ",".join(canonical_extras)) +class Constraint(object): + def __init__(self, specifier, hashes): + # type: (SpecifierSet, Hashes) -> None + self.specifier = specifier + self.hashes = hashes + + @classmethod + def empty(cls): + # type: () -> Constraint + return Constraint(SpecifierSet(), Hashes()) + + @classmethod + def from_ireq(cls, ireq): + # type: (InstallRequirement) -> Constraint + return Constraint(ireq.specifier, ireq.hashes(trust_internet=False)) + + def __nonzero__(self): + # type: () -> bool + return bool(self.specifier) or bool(self.hashes) + + def __bool__(self): + # type: () -> bool + return self.__nonzero__() + + def __and__(self, other): + # type: (InstallRequirement) -> Constraint + if not isinstance(other, InstallRequirement): + return NotImplemented + specifier = self.specifier & other.specifier + hashes = self.hashes & other.hashes(trust_internet=False) + return Constraint(specifier, hashes) + + class Requirement(object): @property def name(self): diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 7209f8c94eb..ed310dd8c7c 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -22,6 +22,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv +from .base import Constraint from .candidates import ( AlreadyInstalledCandidate, EditableCandidate, @@ -152,6 +153,7 @@ def _iter_found_candidates( self, ireqs, # type: Sequence[InstallRequirement] specifier, # type: SpecifierSet + hashes, # type: Hashes ): # type: (...) -> Iterable[Candidate] if not ireqs: @@ -164,7 +166,6 @@ def _iter_found_candidates( template = ireqs[0] name = canonicalize_name(template.req.name) - hashes = Hashes() extras = frozenset() # type: FrozenSet[str] for ireq in ireqs: specifier &= ireq.req.specifier @@ -218,7 +219,7 @@ def _iter_found_candidates( return six.itervalues(candidates) def find_candidates(self, requirements, constraint): - # type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate] + # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate] explicit_candidates = set() # type: Set[Candidate] ireqs = [] # type: List[InstallRequirement] for req in requirements: @@ -231,7 +232,11 @@ def find_candidates(self, requirements, constraint): # If none of the requirements want an explicit candidate, we can ask # the finder for candidates. if not explicit_candidates: - return self._iter_found_candidates(ireqs, constraint) + return self._iter_found_candidates( + ireqs, + constraint.specifier, + constraint.hashes, + ) if constraint: name = explicit_candidates.pop().name diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index b2eb9d06ea5..80577a61c58 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,8 +1,9 @@ -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.providers import AbstractProvider from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint + if MYPY_CHECK_RUNNING: from typing import ( Any, @@ -41,7 +42,7 @@ class PipProvider(AbstractProvider): def __init__( self, factory, # type: Factory - constraints, # type: Dict[str, SpecifierSet] + constraints, # type: Dict[str, Constraint] ignore_dependencies, # type: bool upgrade_strategy, # type: str user_requested, # type: Set[str] @@ -134,7 +135,7 @@ def find_matches(self, requirements): if not requirements: return [] constraint = self._constraints.get( - requirements[0].name, SpecifierSet(), + requirements[0].name, Constraint.empty(), ) candidates = self._factory.find_candidates(requirements, constraint) return reversed(self._sort_matches(candidates)) diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 031d2f107d6..a6e84456650 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -14,12 +14,12 @@ from pip._internal.utils.misc import dist_is_editable from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from .base import Constraint from .factory import Factory if MYPY_CHECK_RUNNING: from typing import Dict, List, Optional, Set, Tuple - from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib.resolvers import Result from pip._vendor.resolvelib.structs import Graph @@ -71,10 +71,11 @@ def __init__( def resolve(self, root_reqs, check_supported_wheels): # type: (List[InstallRequirement], bool) -> RequirementSet - constraints = {} # type: Dict[str, SpecifierSet] + constraints = {} # type: Dict[str, Constraint] user_requested = set() # type: Set[str] requirements = [] for req in root_reqs: + name = canonicalize_name(req.name) if req.constraint: # Ensure we only accept valid constraints problem = check_invalid_constraint_type(req) @@ -82,14 +83,13 @@ def resolve(self, root_reqs, check_supported_wheels): raise InstallationError(problem) if not req.match_markers(): continue - name = canonicalize_name(req.name) if name in constraints: - constraints[name] = constraints[name] & req.specifier + constraints[name] &= req else: - constraints[name] = req.specifier + constraints[name] = Constraint.from_ireq(req) else: if req.user_supplied and req.name: - user_requested.add(canonicalize_name(req.name)) + user_requested.add(name) r = self.factory.make_requirement_from_install_req( req, requested_extras=(), ) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 130f86b622b..703d4b40069 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -85,3 +85,41 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): ) assert message in result.stdout, str(result) + + +def test_new_resolver_hash_intersect_from_constraint(script): + find_links = _create_find_links(script) + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + "base==0.1.0 --hash=sha256:{sdist_hash}".format( + sdist_hash=find_links.sdist_hash, + ), + ) + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ), + ) + + result = script.pip( + "install", + "--use-feature=2020-resolver", + "--no-cache-dir", + "--no-deps", + "--no-index", + "--find-links", find_links.index_html, + "--verbose", + "--constraint", constraints_txt, + "--requirement", requirements_txt, + ) + + message = ( + "Checked 2 links for project 'base' against 1 hashes " + "(1 matches, 0 no digest): discarding 1 non-matches" + ) + assert message in result.stdout, str(result) diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 21de3df4a4f..a03edb6f7c2 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -1,8 +1,7 @@ import pytest -from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.resolvelib import BaseReporter, Resolver -from pip._internal.resolution.resolvelib.base import Candidate +from pip._internal.resolution.resolvelib.base import Candidate, Constraint from pip._internal.utils.urls import path_to_url # NOTE: All tests are prefixed `test_rlr` (for "test resolvelib resolver"). @@ -59,7 +58,7 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory): """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - matches = factory.find_candidates([req], SpecifierSet()) + matches = factory.find_candidates([req], Constraint.empty()) assert len(list(matches)) == match_count @@ -68,7 +67,7 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory): """ for spec, _, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) - for c in factory.find_candidates([req], SpecifierSet()): + for c in factory.find_candidates([req], Constraint.empty()): assert isinstance(c, Candidate) assert req.is_satisfied_by(c)