Complete manylinux support in pex.

docs/api/index.rst

@@ -36,6 +36,13 @@ pex.finders module
     :members:
     :show-inheritance:
 
+pex.glibc module
+----------------
+
+.. automodule:: pex.glibc
+    :members:
+    :show-inheritance:
+
 pex.http module
 --------------------
 
@@ -106,6 +113,13 @@ pex.pex_info module
     :members:
     :show-inheritance:
 
+pex.platforms module
+--------------------
+
+.. automodule:: pex.platforms
+    :members:
+    :show-inheritance:
+
 pex.resolver module
 -------------------
 

docs/buildingpex.rst

@@ -376,11 +376,11 @@ in certain situations when particular extensions may not be necessary to run a p
 The platform to build the pex for. Right now it defaults to the current system, but you can specify
 something like ``linux-x86_64`` or ``macosx-10.6-x86_64``. This will look for bdists for the particular platform.
 
-To build manylinux wheels for specific tags, you can add them to the platform with hyphens like
-``PLATFORM-PYVER-IMPL-ABI``, where ``PLATFORM`` is either ``manylinux1-x86_64`` or ``manylinux1-i686``, ``PYVER``
-is a two-digit string representing the python version (e.g., ``36``), ``IMPL`` is the python implementation
-abbreviation (e.g., ``cp``, ``pp``, ``jp``), and ``ABI`` is the ABI tag (e.g., ``cp36m``, ``cp27mu``, ``abi3``,
-``none``). A complete example: ``manylinux1_x86_64-36-cp-cp36m``.
+To resolve wheels for specific interpreter/platform tags, you can append them to the platform name with hyphens
+like ``PLATFORM-IMPL-PYVER-ABI``, where ``PLATFORM`` is the platform (e.g. ``linux-x86_64``,
+``macosx-10.4-x86_64``), ``IMPL`` is the python implementation abbreviation (e.g. ``cp``, ``pp``, ``jp``), ``PYVER``
+is a two-digit string representing the python version (e.g., ``36``) and ``ABI`` is the ABI tag (e.g., ``cp36m``,
+``cp27mu``, ``abi3``, ``none``). A complete example: ``linux_x86_64-cp-36-cp36m``.
 
 Tailoring PEX execution at runtime
 ----------------------------------

pex/bin/pex.py

@@ -26,10 +26,10 @@
 from pex.interpreter_constraints import validate_constraints
 from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage
-from pex.pep425tags import get_platform
 from pex.pex import PEX
 from pex.pex_bootstrapper import find_compatible_interpreters
 from pex.pex_builder import PEXBuilder
+from pex.platforms import Platform
 from pex.requirements import requirements_from_file
 from pex.resolvable import Resolvable
 from pex.resolver import Unsatisfiable, resolve_multi
@@ -124,6 +124,12 @@ def process_precedence(option, option_str, option_value, parser, builder):
   elif option_str in ('--no-wheel', '--no-use-wheel'):
     setattr(parser.values, option.dest, False)
     builder.no_use_wheel()
+  elif option_str == '--manylinux':
+    setattr(parser.values, option.dest, True)
+    builder.use_manylinux()
+  elif option_str in ('--no-manylinux', '--no-use-manylinux'):
+    setattr(parser.values, option.dest, False)
+    builder.no_use_manylinux()
   else:
     raise OptionValueError
 
@@ -225,6 +231,16 @@ def configure_clp_pex_resolution(parser, builder):
       callback_args=(builder,),
       help='Whether to allow building of distributions from source; Default: allow builds')
 
+  group.add_option(
+      '--manylinux', '--no-manylinux', '--no-use-manylinux',
+      dest='use_manylinux',
+      default=True,
+      action='callback',
+      callback=process_precedence,
+      callback_args=(builder,),
+      help=('Whether to allow resolution of manylinux dists for linux target '
+            'platforms; Default: allow manylinux'))
+
   # Set the pex tool to fetch from PyPI by default if nothing is specified.
   parser.set_default('repos', [PyPIFetcher()])
   parser.add_option_group(group)
@@ -324,18 +340,18 @@ def configure_clp_pex_environment(parser):
 
   group.add_option(
       '--platform',
-      dest='platform',
+      dest='platforms',
       default=[],
       type=str,
       action='append',
       help='The platform for which to build the PEX. This option can be passed multiple times '
-           'to create a multi-platform compatible pex. To build manylinux wheels for specific '
-           'tags, you can add them to the platform with hyphens like PLATFORM-PYVER-IMPL-ABI, '
-           'where PLATFORM is either "manylinux1-x86_64" or "manylinux1-i686", PYVER is a two-'
-           'digit string representing the python version (e.g., 36), IMPL is the python '
-           'implementation abbreviation (e.g., cp, pp, jp), and ABI is the ABI tag (e.g., '
-           'cp36m, cp27mu, abi3, none). For example: manylinux1_x86_64-36-cp-cp36m. '
-           'Default: current platform.')
+           'to create a multi-platform pex. To use wheels for specific interpreter/platform tags'
+           ', you can append them to the platform with hyphens like: PLATFORM-IMPL-PYVER-ABI '
+           '(e.g. "linux_x86_64-cp-27-cp27mu", "macosx_10.12_x86_64-cp-36-cp36m") PLATFORM is '
+           'the host platform e.g. "linux-x86_64", "macosx-10.12-x86_64", etc". IMPL is the '
+           'python implementation abbreviation (e.g. "cp", "pp", "jp"). PYVER is a two-digit '
+           'string representing the python version (e.g. "27", "36"). ABI is the ABI tag '
+           '(e.g. "cp36m", "cp27mu", "abi3", "none"). Default: current platform.')
 
   group.add_option(
       '--interpreter-cache-dir',
@@ -599,10 +615,11 @@ def build_pex(args, options, resolver_option_builder):
     try:
       resolveds = resolve_multi(resolvables,
                                 interpreters=interpreters,
-                                platforms=options.platform,
+                                platforms=options.platforms,
                                 cache=options.cache_dir,
                                 cache_ttl=options.cache_ttl,
-                                allow_prereleases=resolver_option_builder.prereleases_allowed)
+                                allow_prereleases=resolver_option_builder.prereleases_allowed,
+                                use_manylinux=options.use_manylinux)
 
       for dist in resolveds:
         log('  %s' % dist, v=options.verbosity)
@@ -639,6 +656,14 @@ def transform_legacy_arg(arg):
   return arg
 
 
+def _compatible_with_current_platform(platforms):
+  return (
+    not platforms or
+    'current' in platforms or
+    str(Platform.current()) in platforms
+  )
+
+
 def main(args=None):
   args = args[:] if args else sys.argv[1:]
   args = [transform_legacy_arg(arg) for arg in args]
@@ -676,7 +701,7 @@ def main(args=None):
       os.rename(tmp_name, options.pex_name)
       return 0
 
-    if options.platform and get_platform() not in options.platform:
+    if not _compatible_with_current_platform(options.platforms):
       log('WARNING: attempting to run PEX with incompatible platforms!')
 
     pex_builder.freeze()

pex/environment.py

@@ -20,10 +20,9 @@
 from .common import die, open_zip, safe_mkdir, safe_rmtree
 from .interpreter import PythonInterpreter
 from .package import distribution_compatible
-from .pep425tags import get_platform, get_supported
 from .pex_builder import PEXBuilder
 from .pex_info import PexInfo
-from .resolver import platform_to_tags
+from .platforms import Platform
 from .tracer import TRACER
 from .util import CacheHelper, DistributionHelper
 
@@ -112,16 +111,26 @@ def load_internal_cache(cls, pex, pex_info):
         for dist in itertools.chain(*cls.write_zipped_internal_cache(pex, pex_info)):
           yield dist
 
-  def __init__(self, pex, pex_info, supported_tags=None, **kw):
+  def __init__(self, pex, pex_info, interpreter=None, **kw):
     self._internal_cache = os.path.join(pex, pex_info.internal_cache)
     self._pex = pex
     self._pex_info = pex_info
     self._activated = False
     self._working_set = None
+    self._interpreter = interpreter or PythonInterpreter.get()
     self._inherit_path = pex_info.inherit_path
-    self._supported_tags = supported_tags or get_supported()
+    self._supported_tags = []
     super(PEXEnvironment, self).__init__(
-        search_path=[] if pex_info.inherit_path == 'false' else sys.path, **kw)
+      search_path=[] if pex_info.inherit_path == 'false' else sys.path,
+      **kw
+    )
+    self._supported_tags.extend(
+      Platform.create(self.platform).supported_tags(self._interpreter)
+    )
+    TRACER.log(
+      'E: tags for %r x %r -> %s' % (self.platform, self._interpreter, self._supported_tags),
+      V=9
+    )
 
   def update_candidate_distributions(self, distribution_iter):
     for dist in distribution_iter:
@@ -163,7 +172,6 @@ def _resolve(self, working_set, reqs):
     unresolved_reqs = set([req.lower() for req in unresolved_reqs])
 
     if unresolved_reqs:
-      platform_str = '-'.join(platform_to_tags(get_platform(), PythonInterpreter.get()))
       TRACER.log('Unresolved requirements:')
       for req in unresolved_reqs:
         TRACER.log('  - %s' % req)
@@ -174,8 +182,12 @@ def _resolve(self, working_set, reqs):
         for dist in self._pex_info.distributions:
           TRACER.log('  - %s' % dist)
       if not self._pex_info.ignore_errors:
-        die('Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % (
-             platform_str, '\n'.join(map(str, unresolved_reqs))))
+        die(
+          'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % (
+            Platform.current(),
+            '\n'.join(str(r) for r in unresolved_reqs)
+          )
+        )
 
     return resolveds
 

pex/finders.py

@@ -102,6 +102,7 @@ class WheelMetadata(pkg_resources.EggMetadata):
   @classmethod
   def _split_wheelname(cls, wheelname):
     split_wheelname = wheelname.rsplit('-', 4)
+    assert len(split_wheelname) == 5, 'invalid wheel name: %s' % (wheelname)
     split_wheelname[0] = split_wheelname[0].replace('-', '_')
     return '-'.join(split_wheelname[:-3])