From 02ed15bc759054e35e3bd55c82c86cab557effdc Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Thu, 12 Apr 2018 14:27:21 -0700 Subject: [PATCH] Add blacklist handling for skipping requirements in pex resolver (#457) Problem When resolving for a specific interpreter version, the pex resolver fails to resolve universal requirements that transitively require libraries with interpreter constraints that conflict with the target interpreter version (see: https://github.com/Julian/jsonschema/blob/master/setup.py#L43 ). Solution The short term solution is to allow the caller of pex.resolver#resolve to supply a blacklist dict that maps package name -> interpreter constraint, enabling the resolver to skip blacklisted requirement names at resolve time if the target interpreter conforms with the interpreter constraint. The long-term solution is to be addressed by #456. Result It is now possible for systems that build pex files (Pants) via the pex api to blacklist certain requirements that are redundant (backports) or otherwise should not be included in the output pex. --- pex/resolver.py | 46 +++++++++++++++++++++++++++++++++++------- tests/test_resolver.py | 28 ++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/pex/resolver.py b/pex/resolver.py index 31a8e7e29..103327670 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -151,10 +151,11 @@ def filter_packages_by_interpreter(cls, packages, interpreter, platform): return [package for package in packages if package.compatible(interpreter.identity, platform)] - def __init__(self, allow_prereleases=None, interpreter=None, platform=None): + def __init__(self, allow_prereleases=None, interpreter=None, platform=None, pkg_blacklist=None): self._interpreter = interpreter or PythonInterpreter.get() self._platform = platform or Platform.current() self._allow_prereleases = allow_prereleases + self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {} def package_iterator(self, resolvable, existing=None): if existing: @@ -180,6 +181,12 @@ def build(self, package, options): 'Could not get distribution for %s on platform %s.' % (package, self._platform)) return dist + def _resolvable_is_blacklisted(self, resolvable_name): + return ( + resolvable_name in self._blacklist and + self._interpreter.identity.matches(self._blacklist[resolvable_name]) + ) + def resolve(self, resolvables, resolvable_set=None): resolvables = [(resolvable, None) for resolvable in resolvables] resolvable_set = resolvable_set or _ResolvableSet() @@ -193,7 +200,11 @@ def resolve(self, resolvables, resolvable_set=None): if resolvable in processed_resolvables: continue packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name)) - resolvable_set.merge(resolvable, packages, parent) + + # TODO: Remove blacklist strategy in favor of smart requirement handling + # https://github.com/pantsbuild/pex/issues/456 + if not self._resolvable_is_blacklisted(resolvable.name): + resolvable_set.merge(resolvable, packages, parent) processed_resolvables.add(resolvable) built_packages = {} @@ -299,7 +310,8 @@ def resolve(requirements, precedence=None, cache=None, cache_ttl=None, - allow_prereleases=None): + allow_prereleases=None, + pkg_blacklist=None): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either @@ -329,6 +341,14 @@ def resolve(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -367,11 +387,13 @@ def resolve(requirements, cache_ttl, allow_prereleases=allow_prereleases, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) else: resolver = Resolver(allow_prereleases=allow_prereleases, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) return resolver.resolve(resolvables_from_iterable(requirements, builder)) @@ -384,7 +406,8 @@ def resolve_multi(requirements, precedence=None, cache=None, cache_ttl=None, - allow_prereleases=None): + allow_prereleases=None, + pkg_blacklist=None): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -415,6 +438,14 @@ def resolve_multi(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -435,7 +466,8 @@ def resolve_multi(requirements, precedence, cache, cache_ttl, - allow_prereleases): + allow_prereleases, + pkg_blacklist=pkg_blacklist): if resolvable not in seen: seen.add(resolvable) yield resolvable diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 6ae290304..66d28efea 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -2,17 +2,20 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys import time import pytest from twitter.common.contextutil import temporary_dir from pex.common import safe_copy +from pex.compatibility import PY2 from pex.crawler import Crawler from pex.fetcher import Fetcher +from pex.interpreter import PythonInterpreter from pex.package import EggPackage, SourcePackage from pex.resolvable import ResolvableRequirement -from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi +from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.testing import make_sdist @@ -349,3 +352,26 @@ def test_resolvable_set_built(): updated_rs.merge(rq, [binary_pkg]) assert updated_rs.get('foo') == set([binary_pkg]) assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)] + + +def test_resolver_blacklist(): + if PY2: + blacklist = {'project2': '<3'} + required_project = "project2;python_version>'3'" + else: + blacklist = {'project2': '>3'} + required_project = "project2;python_version<'3'" + + project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project]) + project2 = make_sdist(name='project2', version='1.1.0') + + with temporary_dir() as td: + safe_copy(project1, os.path.join(td, os.path.basename(project1))) + safe_copy(project2, os.path.join(td, os.path.basename(project2))) + fetchers = [Fetcher([td])] + + dists = resolve(['project1'], fetchers=fetchers) + assert len(dists) == 2 + + dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist) + assert len(dists) == 1