Make --platform/--interpreter multi-options

.travis.yml

@@ -2,7 +2,7 @@
 # which should provide faster startup times and beefier
 # "machines".
 # See: http://docs.travis-ci.com/user/workers/container-based-infrastructure/
-sudo: false
+sudo: required
 
 language: python
 python: 2.7
@@ -21,5 +21,7 @@ env:
   - TOXENV=isort-check
 install:
   - pip install tox
+  - sudo apt-get install python3-setuptools
+  - sudo python3 -m easy_install wheel
 script:
   - tox -v

pex/bin/pex.py

@@ -30,7 +30,7 @@
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
 from pex.resolvable import Resolvable
-from pex.resolver import CachingResolver, Resolver, Unsatisfiable
+from pex.resolver import Unsatisfiable, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.tracer import TRACER
 from pex.variables import ENV, Variables
@@ -255,10 +255,12 @@ def configure_clp_pex_environment(parser):
   group.add_option(
       '--python',
       dest='python',
-      default=None,
+      default=[None],
+      type=str,
+      action='append',
       help='The Python interpreter to use to build the pex.  Either specify an explicit '
-           'path to an interpreter, or specify a binary accessible on $PATH. '
-           'Default: Use current interpreter.')
+           'path to an interpreter, or specify a binary accessible on $PATH.  This option '
+           'can be used multiple times.  Default: Use current interpreter.')
 
   group.add_option(
       '--python-shebang',
@@ -271,8 +273,11 @@ def configure_clp_pex_environment(parser):
   group.add_option(
       '--platform',
       dest='platform',
-      default=Platform.current(),
-      help='The platform for which to build the PEX.  Default: %default')
+      default=[Platform.current()],
+      type=str,
+      action='append',
+      help='The platform(s) for which to build the PEX.  This option can be used multiple '
+           'times.  Default: %default')
 
   group.add_option(
       '--interpreter-cache-dir',
@@ -431,35 +436,36 @@ def installer_provider(sdist):
     return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
 
 
-def interpreter_from_options(options):
+def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
   interpreter = None
 
-  if options.python:
-    if os.path.exists(options.python):
-      interpreter = PythonInterpreter.from_binary(options.python)
+  if python_interpreter:
+    if os.path.exists(python_interpreter):
+      interpreter = PythonInterpreter.from_binary(python_interpreter)
     else:
-      interpreter = PythonInterpreter.from_env(options.python)
+      interpreter = PythonInterpreter.from_env(python_interpreter)
     if interpreter is None:
-      die('Failed to find interpreter: %s' % options.python)
+      die('Failed to find interpreter: %s' % python_interpreter)
   else:
     interpreter = PythonInterpreter.get()
 
   with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
-    resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)
+    _resolve_interpreter_func = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
 
     # resolve setuptools
-    interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
+    interpreter = _resolve_interpreter_func(interpreter, SETUPTOOLS_REQUIREMENT)
 
     # possibly resolve wheel
-    if interpreter and options.use_wheel:
-      interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
+    if interpreter and use_wheel:
+      interpreter = _resolve_interpreter_func(interpreter, WHEEL_REQUIREMENT)
 
     return interpreter
 
 
 def build_pex(args, options, resolver_option_builder):
   with TRACER.timed('Resolving interpreter', V=2):
-    interpreter = interpreter_from_options(options)
+    interpreter = get_interpreter(options.python[0], options.interpreter_cache_dir,
+            options.repos, options.use_wheel)
 
   if interpreter is None:
     die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
@@ -473,20 +479,18 @@ def build_pex(args, options, resolver_option_builder):
   pex_info.inherit_path = options.inherit_path
 
   resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
+  interpreters = list()
+  for i in options.python:
+    interpreters.append(get_interpreter(i, options.interpreter_cache_dir, options.repos,
+        options.use_wheel))
 
   for requirements_txt in options.requirement_files:
     resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
 
-  resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
-
-  if options.cache_dir:
-    resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
-  else:
-    resolver = Resolver(**resolver_kwargs)
-
   with TRACER.timed('Resolving distributions'):
     try:
-      resolveds = resolver.resolve(resolvables)
+      resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform,
+              cache=options.cache_dir, cache_ttl=options.cache_ttl)
     except Unsatisfiable as e:
       die(e)
 

pex/installer.py

@@ -91,7 +91,7 @@ def bootstrap_script(self):
       if not path:
         assert not self._strict  # This should be caught by validation
         continue
-      bootstrap_modules.append(self.SETUP_BOOTSTRAP_MODULE % {'path': path, 'module': module})
+      bootstrap_modules.append(self.SETUP_BOOTSTRAP_MODULE % {'path': str(path), 'module': module})
     return '\n'.join(
         [self.SETUP_BOOTSTRAP_HEADER] + bootstrap_modules + [self.SETUP_BOOTSTRAP_FOOTER])
 

pex/resolver.py

@@ -344,3 +344,38 @@ def resolve(
     resolver = Resolver(interpreter=interpreter, platform=platform)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
+
+
+def resolve_multi(requirements,
+                  fetchers=None,
+                  interpreters=None,
+                  platforms=None,
+                  context=None,
+                  precedence=None,
+                  cache=None,
+                  cache_ttl=None):
+
+  """Produce all distributions needed to (recursively) meet `requirements` for multiple
+  interpreters and/or platforms.
+
+  :keyword interpreters: (optional) An iterable of :class:`PythonInterpreter` objects to use
+    for building distributions and for testing distribution compatibility.
+  :keyword platforms: (optional) An iterable of PEP425-compatible platform strings to use for
+    filtering compatible distributions.  If unspecified, the current platform is used, as
+    determined by `Platform.current()`.
+
+  This method is just a wrapper around the :func:`resolver` method for resolving requirements
+  for multiple interpreters and/or platforms.
+  """
+
+  platforms = platforms or [Platform.current()]
+  interpreters = interpreters or [PythonInterpreter.get()]
+
+  resolveds = list()
+  for platform in platforms:
+    for interpreter in interpreters:
+      for r in resolve(requirements, fetchers, interpreter, platform, context,
+                       precedence, cache, cache_ttl):
+        resolveds.append(r)
+
+  return resolveds

tests/test_resolver.py

@@ -8,10 +8,12 @@
 
 from pex.common import safe_copy
 from pex.fetcher import Fetcher
+from pex.interpreter import PythonInterpreter
 from pex.package import EggPackage, SourcePackage
 from pex.resolvable import ResolvableRequirement
-from pex.resolver import Unsatisfiable, _ResolvableSet, resolve
+from pex.resolver import Unsatisfiable, _ResolvableSet, resolve, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
+from pex.interpreter import PythonInterpreter
 from pex.testing import make_sdist
 
 
@@ -24,6 +26,15 @@ def test_empty_resolve():
     assert empty_resolve == []
 
 
+def test_empty_resolve_multi():
+  empty_resolve_multi = resolve_multi([])
+  assert empty_resolve_multi == []
+
+  with temporary_dir() as td:
+    empty_resolve_multi = resolve_multi([], cache=td)
+    assert empty_resolve_multi == []
+
+
 def test_simple_local_resolve():
   project_sdist = make_sdist(name='project')
 
@@ -34,6 +45,41 @@ def test_simple_local_resolve():
     assert len(dists) == 1
 
 
+def test_simple_local_resolve_multi():
+  project_sdist = make_sdist(name='project')
+  interpreters = [PythonInterpreter.from_env('python2'), PythonInterpreter.from_env('python3')]
+  platforms = ['linux-x86_64', 'macosx-10.11-x86_64']
+
+  with temporary_dir() as td:
+    safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist)))
+    fetchers = [Fetcher([td])]
+    dists = resolve_multi(['project'], fetchers=fetchers,
+                          interpreters=interpreters, platforms=platforms)
+    assert len(dists) == 4
+
+
+def test_platform_resolve_multi():
+  project_sdist = make_sdist(name='project')
+  interpreters = [PythonInterpreter.from_env('python2'), PythonInterpreter.from_env('python3')]
+
+  with temporary_dir() as td:
+    safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist)))
+    fetchers = [Fetcher([td])]
+    dists = resolve_multi(['project'], fetchers=fetchers, interpreters=interpreters)
+    assert len(dists) == 2
+
+
+def test_interpreters_resolve_multi():
+  project_sdist = make_sdist(name='project')
+  platforms = ['linux-x86_64', 'macosx-10.11-x86_64']
+
+  with temporary_dir() as td:
+    safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist)))
+    fetchers = [Fetcher([td])]
+    dists = resolve_multi(['project'], fetchers=fetchers, platforms=platforms)
+    assert len(dists) == 2
+
+
 def test_diamond_local_resolve_cached():
   # This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
   project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0'])