diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c4ea84798..38a19eecd 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -661,11 +661,10 @@ def walk_and_do(fn, src_dir): allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) - for resolved_dist in resolveds: - log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), - v=options.verbosity) - pex_builder.add_distribution(resolved_dist.distribution) - pex_builder.add_requirement(resolved_dist.requirement) + for dist in resolveds: + log(' %s' % dist, v=options.verbosity) + pex_builder.add_distribution(dist) + pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) diff --git a/pex/environment.py b/pex/environment.py index c1afbb4aa..9f9c056c0 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -161,9 +161,6 @@ def _resolve(self, working_set, reqs): # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. for req in reqs: - if req.marker and not req.marker.evaluate(): - TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req) - continue with TRACER.timed('Resolving %s' % req, V=2): try: resolveds.update(working_set.resolve([req], env=self)) diff --git a/pex/resolver.py b/pex/resolver.py index 8db6ba5bd..911735517 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -159,10 +159,6 @@ def map_packages(resolved_packages): return _ResolvableSet([map_packages(rp) for rp in self.__tuples]) -class ResolvedDistribution(namedtuple('ResolvedDistribution', 'requirement distribution')): - """A requirement and the resolved distribution that satisfies it.""" - - class Resolver(object): """Interface for resolving resolvable entities into python packages.""" @@ -216,10 +212,12 @@ def expand_platform(): # platform. return expand_platform() - def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None): + def __init__(self, allow_prereleases=None, interpreter=None, platform=None, + pkg_blacklist=None, use_manylinux=None): self._interpreter = interpreter or PythonInterpreter.get() self._platform = self._maybe_expand_platform(self._interpreter, platform) self._allow_prereleases = allow_prereleases + self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {} self._supported_tags = self._platform.supported_tags( self._interpreter, use_manylinux @@ -259,6 +257,12 @@ def build(self, package, options): 'Could not get distribution for %s on platform %s.' % (package, self._platform)) return dist + def _resolvable_is_blacklisted(self, resolvable_name): + return ( + resolvable_name in self._blacklist and + self._interpreter.identity.matches(self._blacklist[resolvable_name]) + ) + def resolve(self, resolvables, resolvable_set=None): resolvables = [(resolvable, None) for resolvable in resolvables] resolvable_set = resolvable_set or _ResolvableSet() @@ -273,7 +277,10 @@ def resolve(self, resolvables, resolvable_set=None): continue packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name)) - resolvable_set.merge(resolvable, packages, parent) + # TODO: Remove blacklist strategy in favor of smart requirement handling + # https://github.com/pantsbuild/pex/issues/456 + if not self._resolvable_is_blacklisted(resolvable.name): + resolvable_set.merge(resolvable, packages, parent) processed_resolvables.add(resolvable) built_packages = {} @@ -320,13 +327,7 @@ def resolve(self, resolvables, resolvable_set=None): continue assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable package = next(iter(packages)) - distribution = distributions[package] - if isinstance(resolvable, ResolvableRequirement): - requirement = resolvable.requirement - else: - requirement = distribution.as_requirement() - dists.append(ResolvedDistribution(requirement=requirement, - distribution=distribution)) + dists.append(distributions[package]) return dists @@ -403,6 +404,7 @@ def resolve(requirements, cache=None, cache_ttl=None, allow_prereleases=None, + pkg_blacklist=None, use_manylinux=None): """Produce all distributions needed to (recursively) meet `requirements` @@ -438,8 +440,16 @@ def resolve(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves. - :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. + :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. @@ -465,10 +475,6 @@ def resolve(requirements, .. versionchanged:: 1.0 ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver` classes. - - .. versionchanged:: 1.5.0 - The ``pkg_blacklist`` has been removed and the return type change to a list of - :class:`ResolvedDistribution`. """ builder = ResolverOptionsBuilder(fetchers=fetchers, @@ -483,12 +489,14 @@ def resolve(requirements, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) else: resolver = Resolver(allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) return resolver.resolve(resolvables_from_iterable(requirements, builder)) @@ -502,6 +510,7 @@ def resolve_multi(requirements, cache=None, cache_ttl=None, allow_prereleases=None, + pkg_blacklist=None, use_manylinux=None): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -533,7 +542,15 @@ def resolve_multi(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. - :yields: All :class:`ResolvedDistribution` instances meeting ``requirements``. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 + :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. @@ -554,6 +571,7 @@ def resolve_multi(requirements, cache, cache_ttl, allow_prereleases, + pkg_blacklist=pkg_blacklist, use_manylinux=use_manylinux): if resolvable not in seen: seen.add(resolvable) diff --git a/tests/test_environment.py b/tests/test_environment.py index 4cdefa3bc..00479eda7 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -127,21 +127,20 @@ def bad_interpreter(include_site_extras=True): # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil. for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): - for resolved_dist in resolver.resolve([requirement], - cache=cache, - # We can't use wheels since we're bootstrapping them. - precedence=(SourcePackage, EggPackage), - interpreter=interpreter): - dist = resolved_dist.distribution + for dist in resolver.resolve([requirement], + cache=cache, + # We can't use wheels since we're bootstrapping them. + precedence=(SourcePackage, EggPackage), + interpreter=interpreter): interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), temporary_filename()) as (pb, pex_file): - for resolved_dist in resolver.resolve(['psutil==5.4.3'], - cache=cache, - precedence=(SourcePackage, WheelPackage), - interpreter=interpreter): - pb.add_dist_location(resolved_dist.distribution.location) + for dist in resolver.resolve(['psutil==5.4.3'], + cache=cache, + precedence=(SourcePackage, WheelPackage), + interpreter=interpreter): + pb.add_dist_location(dist.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. diff --git a/tests/test_integration.py b/tests/test_integration.py index 78116aee4..6b2d86c96 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,7 +12,7 @@ import pytest from twitter.common.contextutil import environment_as, temporary_dir -from pex.compatibility import WINDOWS, to_bytes +from pex.compatibility import WINDOWS from pex.installer import EggInstaller from pex.pex_bootstrapper import get_pex_info from pex.testing import ( @@ -1107,39 +1107,3 @@ def test_setup_interpreter_constraint(): '-o', pex]) results.assert_success() subprocess.check_call([pex, '-c', 'import jsonschema']) - - -@pytest.mark.skipif(IS_PYPY, - reason='Our pyenv interpreter setup fails under pypy: ' - 'https://github.com/pantsbuild/pex/issues/477') -def test_setup_python_multiple(): - py27_interpreter = ensure_python_interpreter(PY27) - py36_interpreter = ensure_python_interpreter(PY36) - with temporary_dir() as out: - pex = os.path.join(out, 'pex.pex') - results = run_pex_command(['jsonschema==2.6.0', - '--disable-cache', - '--python-shebang=#!/usr/bin/env python', - '--python={}'.format(py27_interpreter), - '--python={}'.format(py36_interpreter), - '-o', pex]) - results.assert_success() - - pex_program = [pex, '-c'] - py2_only_program = pex_program + ['import functools32'] - both_program = pex_program + [ - 'import jsonschema, os, sys; print(os.path.realpath(sys.executable))' - ] - - with environment_as(PATH=os.path.dirname(py27_interpreter)): - subprocess.check_call(py2_only_program) - - stdout = subprocess.check_output(both_program) - assert to_bytes(os.path.realpath(py27_interpreter)) == stdout.strip() - - with environment_as(PATH=os.path.dirname(py36_interpreter)): - with pytest.raises(subprocess.CalledProcessError): - subprocess.check_call(py2_only_program) - - stdout = subprocess.check_output(both_program) - assert to_bytes(os.path.realpath(py36_interpreter)) == stdout.strip() diff --git a/tests/test_pex.py b/tests/test_pex.py index 5ec6ac1f2..20a926499 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -419,8 +419,7 @@ def test_execute_interpreter_file_program(): def test_pex_run_custom_setuptools_useable(): with temporary_dir() as resolve_cache: - dists = [resolved_dist.distribution - for resolved_dist in resolve(['setuptools==36.2.7'], cache=resolve_cache)] + dists = resolve(['setuptools==36.2.7'], cache=resolve_cache) with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, @@ -441,13 +440,11 @@ def test_pex_run_conflicting_custom_setuptools_useable(): # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py with temporary_dir() as resolve_cache: - dists = [resolved_dist.distribution - for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] + dists = resolve(['setuptools==20.3.1'], cache=resolve_cache) interpreter = PythonInterpreter.from_binary(sys.executable, path_extras=[dist.location for dist in dists], include_site_extras=False) - dists = [resolved_dist.distribution - for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)] + dists = resolve(['setuptools==40.4.3'], cache=resolve_cache) with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, diff --git a/tests/test_resolver.py b/tests/test_resolver.py index d6f3e3ba6..ee35e041d 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -8,11 +8,12 @@ from twitter.common.contextutil import temporary_dir from pex.common import safe_copy +from pex.compatibility import PY2 from pex.crawler import Crawler from pex.fetcher import Fetcher from pex.package import EggPackage, SourcePackage from pex.resolvable import ResolvableRequirement -from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi +from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.testing import make_sdist @@ -32,8 +33,8 @@ def test_simple_local_resolve(): with temporary_dir() as td: safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist))) fetchers = [Fetcher([td])] - resolved_dists = list(resolve_multi(['project'], fetchers=fetchers)) - assert len(resolved_dists) == 1 + dists = list(resolve_multi(['project'], fetchers=fetchers)) + assert len(dists) == 1 def test_diamond_local_resolve_cached(): @@ -46,10 +47,10 @@ def test_diamond_local_resolve_cached(): safe_copy(sdist, os.path.join(dd, os.path.basename(sdist))) fetchers = [Fetcher([dd])] with temporary_dir() as cd: - resolved_dists = list( + dists = list( resolve_multi(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(resolved_dists) == 2 + assert len(dists) == 2 def test_cached_dependency_pinned_unpinned_resolution_multi_run(): @@ -63,39 +64,39 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): fetchers = [Fetcher([td])] with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache - resolved_dists = list( + dists = list( resolve_multi(['project', 'project==1.0.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(resolved_dists) == 1 - assert resolved_dists[0].distribution.version == '1.0.0' + assert len(dists) == 1 + assert dists[0].version == '1.0.0' # This simulates separate invocations of pex but allows us to keep the same tmp cache dir Crawler.reset_cache() # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td - resolved_dists = list( + dists = list( resolve_multi(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(resolved_dists) == 1 - assert resolved_dists[0].distribution.version == '1.1.0' + assert len(dists) == 1 + assert dists[0].version == '1.1.0' # Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact # resolvable should pull from pypi as well since inexact will and the resulting # resolvable_set.merge() would fail. Crawler.reset_cache() time.sleep(1) - resolved_dists = list( + dists = list( resolve_multi(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1) ) - assert len(resolved_dists) == 1 - assert resolved_dists[0].distribution.version == '1.1.0' + assert len(dists) == 1 + assert dists[0].version == '1.1.0' def test_ambiguous_transitive_resolvable(): @@ -110,14 +111,14 @@ def test_ambiguous_transitive_resolvable(): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: - resolved_dists = list( + dists = list( resolve_multi(['foo', 'bar'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(resolved_dists) == 2 - assert resolved_dists[0].distribution.version == '1.0.0' + assert len(dists) == 2 + assert dists[0].version == '1.0.0' def test_resolve_prereleases(): @@ -130,12 +131,12 @@ def test_resolve_prereleases(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - resolved_dists = list( + dists = list( resolve_multi(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(resolved_dists) - resolved_dist = resolved_dists[0] - assert expected_version == resolved_dist.distribution.version + assert 1 == len(dists) + dist = dists[0] + assert expected_version == dist.version assert_resolve('2.0.0') assert_resolve('2.0.0', allow_prereleases=False) @@ -153,12 +154,12 @@ def test_resolve_prereleases_cached(): with temporary_dir() as cd: def assert_resolve(dep, expected_version, **resolve_kwargs): - resolved_dists = list( + dists = list( resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs) ) - assert 1 == len(resolved_dists) - resolved_dist = resolved_dists[0] - assert expected_version == resolved_dist.distribution.version + assert 1 == len(dists) + dist = dists[0] + assert expected_version == dist.version Crawler.reset_cache() @@ -185,12 +186,12 @@ def test_resolve_prereleases_and_no_version(): fetchers = [Fetcher([td])] def assert_resolve(deps, expected_version, **resolve_kwargs): - resolved_dists = list( + dists = list( resolve_multi(deps, fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(resolved_dists) - resolved_dist = resolved_dists[0] - assert expected_version == resolved_dist.distribution.version + assert 1 == len(dists) + dist = dists[0] + assert expected_version == dist.version # When allow_prereleases is specified, the requirement (from two dependencies) # for a specific pre-release version and no version specified, accepts the pre-release @@ -216,14 +217,14 @@ def test_resolve_prereleases_multiple_set(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - resolved_dists = list( + dists = list( resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'], fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(resolved_dists) - resolved_dist = resolved_dists[0] - assert expected_version == resolved_dist.distribution.version + assert 1 == len(dists) + dist = dists[0] + assert expected_version == dist.version # This should resolve with explicit prerelease being set or implicitly. assert_resolve('3.0.0rc4', allow_prereleases=True) @@ -349,3 +350,26 @@ def test_resolvable_set_built(): updated_rs.merge(rq, [binary_pkg]) assert updated_rs.get('foo') == set([binary_pkg]) assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)] + + +def test_resolver_blacklist(): + if PY2: + blacklist = {'project2': '<3'} + required_project = "project2;python_version>'3'" + else: + blacklist = {'project2': '>3'} + required_project = "project2;python_version<'3'" + + project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project]) + project2 = make_sdist(name='project2', version='1.1.0') + + with temporary_dir() as td: + safe_copy(project1, os.path.join(td, os.path.basename(project1))) + safe_copy(project2, os.path.join(td, os.path.basename(project2))) + fetchers = [Fetcher([td])] + + dists = resolve(['project1'], fetchers=fetchers) + assert len(dists) == 2 + + dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist) + assert len(dists) == 1