diff --git a/.travis.yml b/.travis.yml index d991376950c..c14d354c145 100644 --- a/.travis.yml +++ b/.travis.yml @@ -476,6 +476,11 @@ matrix: language: generic env: - SHARD="Rust + Platform-specific Tests OSX" + # Specifically avoid the OSX provided 2.7.10 under xcode8.3 since it returns a platform + # of `macosx-*-intel` where the `intel` suffix is bogus but pex has not yet been taught to + # deal with this. Can be removed when this issue is resolved: + # https://github.com/pantsbuild/pex/issues/523 + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>2.7.10,<3']" before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse before_script: diff --git a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py index 5222edd0811..9115e4e12fa 100644 --- a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py +++ b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py @@ -97,7 +97,7 @@ def _run_mypy(self, py3_interpreter, mypy_args, **kwargs): path = os.path.realpath(os.path.join(self.workdir, str(py3_interpreter.identity), mypy_version)) if not os.path.isdir(path): self.merge_pexes(path, pex_info, py3_interpreter, [mypy_requirement_pex]) - pex = WrappedPEX(PEX(path, py3_interpreter), py3_interpreter) + pex = WrappedPEX(PEX(path, py3_interpreter)) return pex.run(mypy_args, **kwargs) def execute(self): diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py index 904a46bf491..bb70bd8e7a8 100644 --- a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py +++ b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py @@ -157,7 +157,7 @@ def _compile_target(self, vt): exec_pex = PEX(exec_pex_path, interpreter) extra_pex_paths = [pex.path() for pex in filter(None, [reqs_pex, srcs_pex])] - pex = WrappedPEX(exec_pex, interpreter, extra_pex_paths) + pex = WrappedPEX(exec_pex, extra_pex_paths) with self.context.new_workunit(name='eval', labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN, diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD b/examples/src/python/example/python_distribution/hello/setup_requires/BUILD index 4245e88e59f..ca852ee8104 100644 --- a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD +++ b/examples/src/python/example/python_distribution/hello/setup_requires/BUILD @@ -9,6 +9,7 @@ python_dist( sources=[ 'hello_package/hello.py', 'hello_package/__init__.py', + 'setup.cfg', 'setup.py' ], setup_requires=[ diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg new file mode 100644 index 00000000000..3480374bc2f --- /dev/null +++ b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 \ No newline at end of file diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index b8eb9eea2ec..01340de0882 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -13,7 +13,7 @@ from pex.resolver import resolve from pex.variables import Variables -from pants.backend.python.pex_util import create_bare_interpreter, get_local_platform +from pants.backend.python.pex_util import create_bare_interpreter, expand_and_maybe_adjust_platform from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError from pants.process.lock import OwnerPrintingInterProcessFileLock @@ -222,7 +222,11 @@ def _resolve_and_link(self, interpreter, requirement, target_link): distributions = resolve(requirements=[requirement], fetchers=self._python_repos.get_fetchers(), interpreter=interpreter, - platform=get_local_platform(), + platform=expand_and_maybe_adjust_platform( + interpreter=interpreter, + # The local interpreter cache is, by definition, composed of + # interpreters for the 'current' platform. + platform='current'), context=self._python_repos.get_network_context(), precedence=precedence) if not distributions: diff --git a/src/python/pants/backend/python/pex_util.py b/src/python/pants/backend/python/pex_util.py index 7a2841b41ec..a93237de5ba 100644 --- a/src/python/pants/backend/python/pex_util.py +++ b/src/python/pants/backend/python/pex_util.py @@ -4,10 +4,15 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import logging + from pex.interpreter import PythonInterpreter from pex.platforms import Platform +logger = logging.getLogger(__name__) + + def create_bare_interpreter(binary_path): """Creates an interpreter for python binary at the given path. @@ -22,13 +27,61 @@ def create_bare_interpreter(binary_path): return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None) -def get_local_platform(): - """Returns the name of the local platform; eg: 'linux_x86_64' or 'macosx_10_8_x86_64'. +def _interpreter_str(interp): + ident = interp.identity + return ('PythonInterpreter({binary!r}, {identity!r} with extended info: ' + '(abbr_impl: {abbr_impl!r}, impl_ver: {impl_ver!r}, abi_tag: {abi_tag!r}))' + .format(binary=interp.binary, + identity=ident, + abbr_impl=ident.abbr_impl, + impl_ver=ident.impl_ver, + abi_tag=ident.abi_tag)) + + +def expand_and_maybe_adjust_platform(interpreter, platform): + """Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform. - :returns: The local platform name. - :rtype: str + :param interpreter: The target interpreter for the given `platform`. + :type interpreter: :class:`pex.interpreter.PythonInterpreter` + :param platform: The platform name to expand and maybe adjust. + :type platform: text + :returns: The `platform`, potentially adjusted. + :rtype: :class:`pex.platforms.Platform` """ - # TODO(John Sirois): Kill some or all usages when https://github.com/pantsbuild/pex/issues/511 - # is fixed. - current_platform = Platform.current() - return current_platform.platform + # TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed. + cur_plat = Platform.current() + + if cur_plat.platform != Platform.create(platform).platform: + # IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'. + return Platform.create(platform) + + ii = interpreter.identity + if (ii.abbr_impl, ii.impl_ver, ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi): + # IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or + # 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given + # interpreter exactly. + return cur_plat + + # Otherwise we need to adjust the platform to match a local interpreter different from the + # currently executing interpreter. + interpreter_platform = Platform(platform=cur_plat.platform, + impl=ii.abbr_impl, + version=ii.impl_ver, + abi=ii.abi_tag) + + logger.debug(""" +Modifying given platform of {given_platform!r}: +Using the current platform of {current_platform!r} +Under current interpreter {current_interpreter!r} + +To match given interpreter {given_interpreter!r}. + +Calculated platform: {calculated_platform!r}""".format( + given_platform=platform, + current_platform=cur_plat, + current_interpreter=_interpreter_str(PythonInterpreter.get()), + given_interpreter=_interpreter_str(interpreter), + calculated_platform=interpreter_platform) + ) + + return interpreter_platform diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index 9f6dd4d00d4..0d21cb2083a 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -205,10 +205,8 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t 'Installing setup requirements: {}\n\n' .format([req.key for req in setup_reqs_to_resolve])) - cur_platforms = ['current'] if is_platform_specific else None - setup_requires_site_dir = ensure_setup_requires_site_dir( - setup_reqs_to_resolve, interpreter, setup_requires_dir, platforms=cur_platforms) + setup_reqs_to_resolve, interpreter, setup_requires_dir, platforms=['current']) if setup_requires_site_dir: self.context.log.debug('Setting PYTHONPATH with setup_requires site directory: {}' .format(setup_requires_site_dir)) @@ -226,8 +224,9 @@ def _create_dist(self, dist_tgt, dist_target_dir, interpreter, self._copy_sources(dist_tgt, dist_target_dir) setup_runner = SetupPyRunner.for_bdist_wheel( - dist_target_dir, - is_platform_specific=is_platform_specific) + source_dir=dist_target_dir, + is_platform_specific=is_platform_specific, + interpreter=interpreter) with environment_as(**setup_py_execution_environment.as_environment()): # Build a whl using SetupPyRunner and return its absolute path. diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index 1a0002c3238..eb3e2190c7f 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -11,7 +11,7 @@ from pex.resolver import resolve from twitter.common.collections import OrderedSet -from pants.backend.python.pex_util import get_local_platform +from pants.backend.python.pex_util import expand_and_maybe_adjust_platform from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary @@ -155,7 +155,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links): requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, - platform=get_local_platform() if platform == 'current' else platform, + platform=expand_and_maybe_adjust_platform(interpreter=interpreter, platform=platform), context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, diff --git a/src/python/pants/backend/python/tasks/python_execution_task_base.py b/src/python/pants/backend/python/tasks/python_execution_task_base.py index 8ee154c1f45..7d6f79d9dae 100644 --- a/src/python/pants/backend/python/tasks/python_execution_task_base.py +++ b/src/python/pants/backend/python/tasks/python_execution_task_base.py @@ -119,4 +119,4 @@ def create_pex(self, pex_info=None): extra_file.add_to(builder) builder.freeze() - return WrappedPEX(PEX(path, interpreter), interpreter) + return WrappedPEX(PEX(path, interpreter)) diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index f83914396f8..f917c606199 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -13,6 +13,7 @@ from builtins import map, object, str, zip from collections import OrderedDict, defaultdict +from pex import pep425tags from pex.compatibility import string, to_bytes from pex.installer import InstallerBase, Packager from pex.interpreter import PythonInterpreter @@ -21,7 +22,6 @@ from wheel.install import WheelFile from pants.backend.native.config.environment import CCompiler, CppCompiler, Linker, Platform -from pants.backend.python.pex_util import get_local_platform from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget @@ -63,9 +63,7 @@ class SetupPyRunner(InstallerBase): def for_bdist_wheel(cls, source_dir, is_platform_specific, **kw): cmd = ['bdist_wheel'] if is_platform_specific: - cmd.extend(['--plat-name', get_local_platform()]) - else: - cmd.append('--universal') + cmd.extend(['--plat-name', pep425tags.get_platform()]) cmd.extend(['--dist-dir', cls.DIST_DIR]) return cls(source_dir, cmd, **kw) diff --git a/src/python/pants/backend/python/tasks/wrapped_pex.py b/src/python/pants/backend/python/tasks/wrapped_pex.py index 4a128f7baab..1676f490154 100644 --- a/src/python/pants/backend/python/tasks/wrapped_pex.py +++ b/src/python/pants/backend/python/tasks/wrapped_pex.py @@ -4,10 +4,14 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import logging from builtins import object from copy import copy +logger = logging.getLogger(__name__) + + class WrappedPEX(object): """Wrapper around a PEX that exposes only its run() method. @@ -15,46 +19,53 @@ class WrappedPEX(object): """ _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH' + _PEX_PYTHON_PATH_ENV_VAR_NAME = 'PEX_PYTHON_PATH' - # TODO(benjy): I think we can get rid of the interpreter argument. - # In all cases it appears to be set to pex.interpreter. - def __init__(self, pex, interpreter, extra_pex_paths=None): + def __init__(self, pex, extra_pex_paths=None): """ :param pex: The main pex we wrap. - :param interpreter: The interpreter the main pex will run on. :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism. """ self._pex = pex - self._interpreter = interpreter self._extra_pex_paths = extra_pex_paths @property def interpreter(self): - return self._interpreter + return self._pex._interpreter def path(self): return self._pex.path() def cmdline(self, args=()): cmdline = ' '.join(self._pex.cmdline(args)) + + def render_env_var(key, value): + return '{key}={value}'.format(key=key, value=value) + + env_vars = [(self._PEX_PYTHON_PATH_ENV_VAR_NAME, self._pex._interpreter.binary)] + pex_path = self._pex_path() if pex_path: - return '{env_var_name}={pex_path} {cmdline}'.format(env_var_name=self._PEX_PATH_ENV_VAR_NAME, - pex_path=pex_path, - cmdline=cmdline) - else: - return cmdline + env_vars.append((self._PEX_PATH_ENV_VAR_NAME, pex_path)) + + return '{execution_control_env_vars} {cmdline}'.format( + execution_control_env_vars=' '.join(render_env_var(k, v) for k, v in env_vars), + cmdline=cmdline + ) def run(self, *args, **kwargs): + env = copy(kwargs.pop('env', {})) + + # Hack around bug in PEX where custom interpreters are not forwarded to PEXEnvironments. + # TODO(John Sirois): Remove when https://github.com/pantsbuild/pex/issues/522 is fixed. + env[self._PEX_PYTHON_PATH_ENV_VAR_NAME] = self._pex._interpreter.binary + pex_path = self._pex_path() if pex_path: - kwargs_copy = copy(kwargs) - env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {} - env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path() - kwargs_copy['env'] = env - return self._pex.run(*args, **kwargs_copy) - else: - return self._pex.run(*args, **kwargs) + env[self._PEX_PATH_ENV_VAR_NAME] = pex_path + + logger.debug('Executing WrappedPEX using: {}'.format(self.cmdline(args=tuple(*args)))) + return self._pex.run(*args, env=env, **kwargs) def _pex_path(self): if self._extra_pex_paths: diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index f8a2aac928d..c013d962389 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -88,10 +88,12 @@ python_tests( sources=globs('*_integration.py', exclude=python_native_code_test_files), dependencies=[ '3rdparty/python:pex', + 'src/python/pants/base:build_environment', 'src/python/pants/util:contextutil', 'src/python/pants/util:process_handler', - 'tests/python/pants_test/backend/python:pants_requirement_integration_test_base', 'tests/python/pants_test:int-test', + 'tests/python/pants_test/backend/python:pants_requirement_integration_test_base', + 'tests/python/pants_test/backend/python/tasks:python_task_test_base', ], tags={'integration'}, timeout=2400 diff --git a/tests/python/pants_test/backend/python/tasks/python_task_test_base.py b/tests/python/pants_test/backend/python/tasks/python_task_test_base.py index 4199c8db622..c8f79b866f3 100644 --- a/tests/python/pants_test/backend/python/tasks/python_task_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/python_task_test_base.py @@ -8,7 +8,8 @@ from builtins import map from textwrap import dedent -from pants.backend.python.pex_util import get_local_platform +from pex import pep425tags + from pants.backend.python.register import build_file_aliases as register_python from pants.build_graph.address import Address from pants_test.backend.python.tasks.interpreter_cache_test_mixin import InterpreterCacheTestMixin @@ -34,7 +35,7 @@ def name_and_platform(whl): def check_wheel_platform_matches_host(wheel_dist): _, _, wheel_platform = name_and_platform(wheel_dist) - return wheel_platform == normalize_platform_tag(get_local_platform()) + return wheel_platform == normalize_platform_tag(pep425tags.get_platform()) class PythonTaskTestBase(InterpreterCacheTestMixin, TaskTestBase):