From c8b5bdf1bbd049708595a3eac088abc63a7e83cf Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Mon, 24 Jul 2017 14:51:22 -0700 Subject: [PATCH 1/3] Multi-interpreter and multi-platform pex creation. --- .gitignore | 1 + pex/bin/pex.py | 72 +++++++++++++++---------- pex/resolver.py | 108 ++++++++++++++++++++++++++++++-------- pex/testing.py | 7 +++ tests/test_integration.py | 33 +++++++++++- tests/test_resolver.py | 62 +++++++++++++++------- 6 files changed, 213 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index 4c7e3a286..1226a163e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ *.pyc *.egg-info +/.cache /venv /build/* /dist/* diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 32eb0a857..92585057c 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -30,7 +30,7 @@ from pex.platforms import Platform from pex.requirements import requirements_from_file from pex.resolvable import Resolvable -from pex.resolver import CachingResolver, Resolver, Unsatisfiable +from pex.resolver import Unsatisfiable, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.tracer import TRACER from pex.variables import ENV, Variables @@ -274,9 +274,12 @@ def configure_clp_pex_environment(parser): group.add_option( '--python', dest='python', - default=None, + default=[], + type='str', + action='append', help='The Python interpreter to use to build the pex. Either specify an explicit ' - 'path to an interpreter, or specify a binary accessible on $PATH. ' + 'path to an interpreter, or specify a binary accessible on $PATH. This option ' + 'can be passed multiple times to create a multi-interpreter compatible pex. ' 'Default: Use current interpreter.') group.add_option( @@ -290,8 +293,11 @@ def configure_clp_pex_environment(parser): group.add_option( '--platform', dest='platform', - default=Platform.current(), - help='The platform for which to build the PEX. Default: %default') + default=[], + type=str, + action='append', + help='The platform for which to build the PEX. This option can be passed multiple times ' + 'to create a multi-platform compatible pex. Default: %default') group.add_option( '--interpreter-cache-dir', @@ -460,39 +466,54 @@ def installer_provider(sdist): return interpreter.with_extra(egg.name, egg.raw_version, egg.path) -def interpreter_from_options(options): +def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel): interpreter = None - if options.python: - if os.path.exists(options.python): - interpreter = PythonInterpreter.from_binary(options.python) + if python_interpreter: + if os.path.exists(python_interpreter): + interpreter = PythonInterpreter.from_binary(python_interpreter) else: - interpreter = PythonInterpreter.from_env(options.python) + interpreter = PythonInterpreter.from_env(python_interpreter) if interpreter is None: - die('Failed to find interpreter: %s' % options.python) + die('Failed to find interpreter: %s' % python_interpreter) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): - resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) + resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel - if interpreter and options.use_wheel: + if interpreter and use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter -def build_pex(args, options, resolver_option_builder): - with TRACER.timed('Resolving interpreter', V=2): - interpreter = interpreter_from_options(options) +def _lowest_version_interpreter(interpreters): + """Given an iterable of interpreters, return the one with the lowest version.""" + lowest = interpreters[0] + for i in interpreters[1:]: + lowest = lowest if lowest < i else i + return lowest - if interpreter is None: + +def build_pex(args, options, resolver_option_builder): + with TRACER.timed('Resolving interpreters', V=2): + interpreters = [ + get_interpreter(interpreter, + options.interpreter_cache_dir, + options.repos, + options.use_wheel) + for interpreter in options.python or [None] + ] + + if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) + interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info @@ -515,16 +536,13 @@ def build_pex(args, options, resolver_option_builder): constraints.append(r) resolvables.extend(constraints) - resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) - - if options.cache_dir: - resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) - else: - resolver = Resolver(**resolver_kwargs) - with TRACER.timed('Resolving distributions'): try: - resolveds = resolver.resolve(resolvables) + resolveds = resolve_multi(resolvables, + interpreters=interpreters, + platforms=options.platform, + cache=options.cache_dir, + cache_ttl=options.cache_ttl) except Unsatisfiable as e: die(e) @@ -585,8 +603,8 @@ def main(args=None): os.rename(tmp_name, options.pex_name) return 0 - if options.platform != Platform.current(): - log('WARNING: attempting to run PEX with differing platform!') + if options.platform and Platform.current() not in options.platform: + log('WARNING: attempting to run PEX with incompatible platforms!') pex_builder.freeze() diff --git a/pex/resolver.py b/pex/resolver.py index d9867565d..31a8e7e29 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -291,17 +291,15 @@ def build(self, package, options): return DistributionHelper.distribution_from_path(target) -def resolve( - requirements, - fetchers=None, - interpreter=None, - platform=None, - context=None, - precedence=None, - cache=None, - cache_ttl=None, - allow_prereleases=None): - +def resolve(requirements, + fetchers=None, + interpreter=None, + platform=None, + context=None, + precedence=None, + cache=None, + cache_ttl=None, + allow_prereleases=None): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either @@ -359,19 +357,85 @@ def resolve( classes. """ - builder = ResolverOptionsBuilder( - fetchers=fetchers, - allow_prereleases=allow_prereleases, - precedence=precedence, - context=context, - ) + builder = ResolverOptionsBuilder(fetchers=fetchers, + allow_prereleases=allow_prereleases, + precedence=precedence, + context=context) if cache: - resolver = CachingResolver( - cache, cache_ttl, - allow_prereleases=allow_prereleases, interpreter=interpreter, platform=platform) + resolver = CachingResolver(cache, + cache_ttl, + allow_prereleases=allow_prereleases, + interpreter=interpreter, + platform=platform) else: - resolver = Resolver( - allow_prereleases=allow_prereleases, interpreter=interpreter, platform=platform) + resolver = Resolver(allow_prereleases=allow_prereleases, + interpreter=interpreter, + platform=platform) return resolver.resolve(resolvables_from_iterable(requirements, builder)) + + +def resolve_multi(requirements, + fetchers=None, + interpreters=None, + platforms=None, + context=None, + precedence=None, + cache=None, + cache_ttl=None, + allow_prereleases=None): + """A generator function that produces all distributions needed to meet `requirements` + for multiple interpreters and/or platforms. + + :param requirements: An iterator of Requirement-like things, either + :class:`pkg_resources.Requirement` objects or requirement strings. + :keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages. If + unspecified, the default is to look for packages on PyPI. + :keyword interpreters: (optional) An iterable of :class:`PythonInterpreter` objects to use + for building distributions and for testing distribution compatibility. + :keyword platforms: (optional) An iterable of PEP425-compatible platform strings to use for + filtering compatible distributions. If unspecified, the current platform is used, as + determined by `Platform.current()`. + :keyword context: (optional) A :class:`Context` object to use for network access. If + unspecified, the resolver will attempt to use the best available network context. + :keyword precedence: (optional) An ordered list of allowable :class:`Package` classes + to be used for producing distributions. For example, if precedence is supplied as + ``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and + eggs will not be used at all. If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and + eggs will be used, but the resolver will not resort to building anything from source. + :keyword cache: (optional) A directory to use to cache distributions locally. + :keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when + resolving requirements. For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is + available in the cache, it will always be used. However, if a non-exact requirement such as + ``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than + cache_ttl seconds that satisfies the requirement, then it will be used. If the distribution + is older than cache_ttl seconds, it will be ignored. If ``cache_ttl`` is not specified, + resolving inexact requirements will always result in making network calls through the + ``context``. + :keyword allow_prereleases: (optional) Include pre-release and development versions. If + unspecified only stable versions will be resolved, unless explicitly included. + :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``. + :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. + :raises Untranslateable: If no compatible distributions could be acquired for + a particular requirement. + """ + + interpreters = interpreters or [PythonInterpreter.get()] + platforms = platforms or [Platform.current()] + + seen = set() + for interpreter in interpreters: + for platform in platforms: + for resolvable in resolve(requirements, + fetchers, + interpreter, + platform, + context, + precedence, + cache, + cache_ttl, + allow_prereleases): + if resolvable not in seen: + seen.add(resolvable) + yield resolvable diff --git a/pex/testing.py b/pex/testing.py index 2480770d0..d49b52f38 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -46,6 +46,13 @@ def random_bytes(length): map(chr, (random.randint(ord('a'), ord('z')) for _ in range(length)))).encode('utf-8') +def get_dep_dist_names_from_pex(pex_path, match_prefix=''): + """Given an on-disk pex, extract all of the unique first-level paths under `.deps`.""" + with zipfile.ZipFile(pex_path) as pex_zip: + dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith('.deps/')) + return set(item for item in dep_gen if item.startswith(match_prefix)) + + @contextlib.contextmanager def temporary_content(content_map, interp=None, seed=31337): """Write content to disk where content is map from string => (int, string). diff --git a/tests/test_integration.py b/tests/test_integration.py index a3d101925..30a1fa678 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,9 +10,20 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller -from pex.testing import run_pex_command, run_simple_pex, run_simple_pex_test, temporary_content +from pex.testing import ( + get_dep_dist_names_from_pex, + run_pex_command, + run_simple_pex, + run_simple_pex_test, + temporary_content +) from pex.util import DistributionHelper, named_temporary_file +NOT_CPYTHON_36 = ( + "hasattr(sys, 'pypy_version_info') or " + "(sys.version_info[0], sys.version_info[1]) != (3, 6)" +) + def test_pex_execute(): body = "print('Hello')" @@ -155,3 +166,23 @@ def do_something(): so, rc = run_simple_pex_test('', env={'PEX_SCRIPT': 'my_app'}, dists=[dist]) assert so.decode('utf-8').strip() == error_msg assert rc == 1 + + +@pytest.mark.skipif(NOT_CPYTHON_36) +def test_pex_multi_resolve(): + """Tests multi-interpreter + multi-platform resolution.""" + with temporary_dir() as output_dir: + pex_path = os.path.join(output_dir, 'pex.pex') + results = run_pex_command(['--disable-cache', + 'lxml==3.8.0', + '--platform=manylinux1-x86_64', + '--platform=macosx-10.6-x86_64', + '--python=python2.7', + '--python=python3.6', + '-o', pex_path]) + results.assert_success() + + included_dists = get_dep_dist_names_from_pex(pex_path, 'lxml') + assert len(included_dists) == 4 + for dist_substr in ('-cp27-', '-cp36-', '-manylinux1_x86_64', '-macosx_'): + assert any(dist_substr in f for f in included_dists) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 5dad6f41f..73fb10f61 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -12,18 +12,18 @@ from pex.fetcher import Fetcher from pex.package import EggPackage, SourcePackage from pex.resolvable import ResolvableRequirement -from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve +from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.testing import make_sdist def test_empty_resolve(): - empty_resolve = resolve([]) - assert empty_resolve == [] + empty_resolve_multi = list(resolve_multi([])) + assert empty_resolve_multi == [] with temporary_dir() as td: - empty_resolve = resolve([], cache=td) - assert empty_resolve == [] + empty_resolve_multi = list(resolve_multi([], cache=td)) + assert empty_resolve_multi == [] def test_simple_local_resolve(): @@ -32,7 +32,7 @@ def test_simple_local_resolve(): with temporary_dir() as td: safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist))) fetchers = [Fetcher([td])] - dists = resolve(['project'], fetchers=fetchers) + dists = list(resolve_multi(['project'], fetchers=fetchers)) assert len(dists) == 1 @@ -46,7 +46,9 @@ def test_diamond_local_resolve_cached(): safe_copy(sdist, os.path.join(dd, os.path.basename(sdist))) fetchers = [Fetcher([dd])] with temporary_dir() as cd: - dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) + dists = list( + resolve_multi(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) + ) assert len(dists) == 2 @@ -61,14 +63,24 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): fetchers = [Fetcher([td])] with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache - dists = resolve(['project', 'project==1.0.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) + dists = list( + resolve_multi(['project', 'project==1.0.0'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) + ) assert len(dists) == 1 assert dists[0].version == '1.0.0' # This simulates separate invocations of pex but allows us to keep the same tmp cache dir Crawler.reset_cache() # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td - dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) + dists = list( + resolve_multi(['project', 'project==1.1.0'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) + ) assert len(dists) == 1 assert dists[0].version == '1.1.0' # Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact @@ -76,8 +88,12 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): # resolvable_set.merge() would fail. Crawler.reset_cache() time.sleep(1) - dists = resolve(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, + dists = list( + resolve_multi(['project', 'project==1.1.0'], + fetchers=fetchers, + cache=cd, cache_ttl=1) + ) assert len(dists) == 1 assert dists[0].version == '1.1.0' @@ -94,8 +110,12 @@ def test_ambiguous_transitive_resolvable(): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: - dists = resolve(['foo', 'bar'], fetchers=fetchers, cache=cd, + dists = list( + resolve_multi(['foo', 'bar'], + fetchers=fetchers, + cache=cd, cache_ttl=1000) + ) assert len(dists) == 2 assert dists[0].version == '1.0.0' @@ -110,7 +130,9 @@ def test_resolve_prereleases(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - dists = resolve(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) + dists = list( + resolve_multi(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) + ) assert 1 == len(dists) dist = dists[0] assert expected_version == dist.version @@ -131,8 +153,9 @@ def test_resolve_prereleases_cached(): with temporary_dir() as cd: def assert_resolve(dep, expected_version, **resolve_kwargs): - dists = resolve( - [dep], cache=cd, cache_ttl=1000, **resolve_kwargs) + dists = list( + resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs) + ) assert 1 == len(dists) dist = dists[0] assert expected_version == dist.version @@ -166,12 +189,11 @@ def test_resolve_prereleases_multiple_set(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - dists = resolve( - [ - 'dep>=3.0.0rc1', - 'dep==3.0.0rc4', - ], - fetchers=fetchers, **resolve_kwargs) + dists = list( + resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'], + fetchers=fetchers, + **resolve_kwargs) + ) assert 1 == len(dists) dist = dists[0] assert expected_version == dist.version From 742189e74047291557f723fc12f409302de3474f Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Mon, 24 Jul 2017 20:42:51 -0700 Subject: [PATCH 2/3] Move generator unwinding into try/except block. --- pex/bin/pex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 92585057c..baac58230 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -543,14 +543,14 @@ def build_pex(args, options, resolver_option_builder): platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl) + + for dist in resolveds: + log(' %s' % dist, v=options.verbosity) + pex_builder.add_distribution(dist) + pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) - for dist in resolveds: - log(' %s' % dist, v=options.verbosity) - pex_builder.add_distribution(dist) - pex_builder.add_requirement(dist.as_requirement()) - if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) From 5c6284dc5269c65d3892a99327a722aee40ce21d Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Mon, 24 Jul 2017 20:50:43 -0700 Subject: [PATCH 3/3] Review feedback. --- pex/bin/pex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index baac58230..dc99b70d0 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -297,14 +297,14 @@ def configure_clp_pex_environment(parser): type=str, action='append', help='The platform for which to build the PEX. This option can be passed multiple times ' - 'to create a multi-platform compatible pex. Default: %default') + 'to create a multi-platform compatible pex. Default: current platform.') group.add_option( '--interpreter-cache-dir', dest='interpreter_cache_dir', default='{pex_root}/interpreters', help='The interpreter cache to use for keeping track of interpreter dependencies ' - 'for the pex tool. [Default: ~/.pex/interpreters]') + 'for the pex tool. Default: `~/.pex/interpreters`.') parser.add_option_group(group) @@ -493,7 +493,7 @@ def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel) def _lowest_version_interpreter(interpreters): - """Given an iterable of interpreters, return the one with the lowest version.""" + """Given a list of interpreters, return the one with the lowest version.""" lowest = interpreters[0] for i in interpreters[1:]: lowest = lowest if lowest < i else i