From 460a9ae9d923793b4a0474d5d58f15254757afb4 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Fri, 20 Oct 2017 12:12:55 -0700 Subject: [PATCH 01/43] Add interpreter constraints option to pex CLI and write data to PEX-INFO; filter interpreters used to build/run the pex based on these constraints. --- pex/bin/pex.py | 17 ++++++++ pex/interpreter_constraints.py | 47 ++++++++++++++++++++++ pex/pex_info.py | 15 +++++++ tests/test_integration.py | 71 ++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100644 pex/interpreter_constraints.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 86a23cccb..2a913da0a 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,6 +23,7 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter +from pex.interpreter_constraints import matched_interpreters, parse_interpreter_constraints from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX @@ -289,6 +290,15 @@ def configure_clp_pex_environment(parser): 'can be passed multiple times to create a multi-interpreter compatible pex. ' 'Default: Use current interpreter.') + group.add_option( + '--interpreter-constraints', + dest='interpreter_constraints', + default='', + type='str', + help='A comma-seperated list of constraints that determine the interpreter compatibility for ' + 'this pex, using the Requirement-style format, e.g. "CPython>=3", or just ">=2.7,<3" ' + 'for requirements agnostic to interpreter class.') + group.add_option( '--python-shebang', dest='python_shebang', @@ -525,6 +535,11 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] + if options.interpreter_constraints: + safe_constraints = options.interpreter_constraints.strip("'").strip('"') + constraints = parse_interpreter_constraints(safe_constraints) + interpreters = list(matched_interpreters(interpreters, constraints, True)) + if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) @@ -544,6 +559,8 @@ def build_pex(args, options, resolver_option_builder): pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path + if options.interpreter_constraints: + pex_info.interpreter_constraints = safe_constraints resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py new file mode 100644 index 000000000..ac523b530 --- /dev/null +++ b/pex/interpreter_constraints.py @@ -0,0 +1,47 @@ +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# A library of functions for filtering Python interpreters based on compatibility constraints + +def _matches(interpreter, filters, match_all=False): + if match_all: + return all(interpreter.identity.matches(filt) for filt in filters) + else: + return any(interpreter.identity.matches(filt) for filt in filters) + + +def _matching(interpreters, filters, match_all=False): + for interpreter in interpreters: + if _matches(interpreter, filters, match_all): + yield interpreter + + +def matched_interpreters(interpreters, filters, match_all=False): + """Given some filters, yield any interpreter that matches at least one of them, or all of them + if match_all is set to True. + + :param interpreters: a list of PythonInterpreter objects for filtering + :param filters: A sequence of strings that constrain the interpreter compatibility for this + pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` + for requirements agnostic to interpreter class. + :param match_all: whether to match against all constraints. Defaults to matching one constraint. + """ + for match in _matching(interpreters, filters, match_all): + yield match + + +def parse_interpreter_constraints(constraints_string): + """Given a single string defining interpreter constraints, separate them into a list of + individual constraint items for PythonIdentity to consume. + + Example: '>=2.7, <3' + Return: ['>=2.7', '<3'] + + Example: 'CPython>=2.7,<3' + Return: ['CPython>=2.7', 'CPython<3'] + """ + if 'CPython' in constraints_string: + return list(map(lambda x: 'CPython' + x.strip() if not 'CPython' in x else x.strip(), + constraints_string.split(','))) + else: + return list(map(lambda x: x.strip(), constraints_string.split(','))) diff --git a/pex/pex_info.py b/pex/pex_info.py index e6a053116..1d11bcb0c 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -195,6 +195,21 @@ def inherit_path(self): def inherit_path(self, value): self._pex_info['inherit_path'] = bool(value) + @property + def interpreter_constraints(self): + """A comma-seperated list of constraints that determine the interpreter compatibility for this + pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'`` + for requirements agnostic to interpreter class. + + This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH + for a list of compatible interpreters. + """ + return self._pex_info.get('interpreter_constraints', False) + + @interpreter_constraints.setter + def interpreter_constraints(self, value): + self._pex_info['interpreter_constraints'] = value + @property def ignore_errors(self): return self._pex_info.get('ignore_errors', False) diff --git a/tests/test_integration.py b/tests/test_integration.py index 58ece3a2c..8dbbdcb25 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,6 +10,7 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller +from pex.pex_bootstrapper import read_pexinfo_from_zip from pex.testing import ( get_dep_dist_names_from_pex, run_pex_command, @@ -326,3 +327,73 @@ def test_pex_path_in_pex_info_and_env(): stdout, rc = run_simple_pex(pex_out_path, [test_file_path], env=env) assert rc == 0 assert stdout == b'Success!\n' + + +def test_interpreter_constraints_to_pex_info(): + with temporary_dir() as output_dir: + # constraint without interpreter class + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints=">=2.7,<3"', + '-o', pex_out_path]) + if sys.version_info[0] == 3: + assert res.return_code == 102 + else: + res.assert_success() + pex_info = read_pexinfo_from_zip(pex_out_path) + assert '>=2.7,<3' in pex_info + + # constraint with interpreter class + pex_out_path = os.path.join(output_dir, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints="CPython>=2.7,<3"', + '-o', pex_out_path]) + if sys.version_info[0] == 3: + assert res.return_code == 102 + else: + res.assert_success() + pex_info = read_pexinfo_from_zip(pex_out_path) + assert 'CPython>=2.7,<3' in pex_info + + +@pytest.mark.skipif(NOT_CPYTHON_36) +def test_interpreter_constraints_to_pex_info_py36(): + with temporary_dir() as output_dir: + # constraint without interpreter class + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints=">=3"', + '-o', pex_out_path]) + res.assert_success() + pex_info = read_pexinfo_from_zip(pex_out_path) + assert b'>=3' in pex_info + + # constraint with interpreter class + pex_out_path = os.path.join(output_dir, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints="CPython>=3"', + '-o', pex_out_path]) + res.assert_success() + pex_info = read_pexinfo_from_zip(pex_out_path) + assert b'CPython>=3' in pex_info + + +def test_resolve_interpreter_with_constraints_option(): + with temporary_dir() as output_dir: + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints=">=2.7,<3"', + '-o', pex_out_path]) + if sys.version_info[0] == 3: + assert res.return_code == 102 + else: + res.assert_success() + + pex_out_path = os.path.join(output_dir, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints=">3"', + '-o', pex_out_path]) + if sys.version_info[0] == 3: + res.assert_success() + else: + assert res.return_code == 102 From 45e4640230742ec616a26ecccba61f9001168938 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 23 Oct 2017 11:13:00 -0700 Subject: [PATCH 02/43] Improve integration tests --- tests/test_integration.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 8dbbdcb25..df5943e69 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,6 +11,7 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller from pex.pex_bootstrapper import read_pexinfo_from_zip +from pex.pex_info import PexInfo from pex.testing import ( get_dep_dist_names_from_pex, run_pex_command, @@ -340,20 +341,20 @@ def test_interpreter_constraints_to_pex_info(): assert res.return_code == 102 else: res.assert_success() - pex_info = read_pexinfo_from_zip(pex_out_path) - assert '>=2.7,<3' in pex_info + pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + assert '>=2.7,<3' == pex_info.interpreter_constraints # constraint with interpreter class pex_out_path = os.path.join(output_dir, 'pex2.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraints="CPython>=2.7,<3"', '-o', pex_out_path]) - if sys.version_info[0] == 3: + if sys.version_info[0] == 3 or hasattr(sys, 'pypy_version_info'): assert res.return_code == 102 else: res.assert_success() - pex_info = read_pexinfo_from_zip(pex_out_path) - assert 'CPython>=2.7,<3' in pex_info + pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + assert 'CPython>=2.7,<3' == pex_info.interpreter_constraints @pytest.mark.skipif(NOT_CPYTHON_36) @@ -365,8 +366,8 @@ def test_interpreter_constraints_to_pex_info_py36(): '--interpreter-constraints=">=3"', '-o', pex_out_path]) res.assert_success() - pex_info = read_pexinfo_from_zip(pex_out_path) - assert b'>=3' in pex_info + pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + assert '>=3' == pex_info.interpreter_constraints # constraint with interpreter class pex_out_path = os.path.join(output_dir, 'pex2.pex') @@ -374,8 +375,8 @@ def test_interpreter_constraints_to_pex_info_py36(): '--interpreter-constraints="CPython>=3"', '-o', pex_out_path]) res.assert_success() - pex_info = read_pexinfo_from_zip(pex_out_path) - assert b'CPython>=3' in pex_info + pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + assert 'CPython>=3' == pex_info.interpreter_constraints def test_resolve_interpreter_with_constraints_option(): From 7ae3b612a65a064220120d6d6154e988c3052715 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 23 Oct 2017 16:03:00 -0700 Subject: [PATCH 03/43] Move helper function to interpreter constraints lib, refactor lib, and add a unit test for new method --- pex/bin/pex.py | 18 +++++++----------- pex/interpreter_constraints.py | 27 +++++++++++++++++++-------- tests/test_interpreter_constraints.py | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+), 19 deletions(-) create mode 100644 tests/test_interpreter_constraints.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 2a913da0a..d32504eee 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,7 +23,11 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter -from pex.interpreter_constraints import matched_interpreters, parse_interpreter_constraints +from pex.interpreter_constraints import ( + lowest_version_interpreter, + matched_interpreters, + parse_interpreter_constraints +) from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX @@ -517,14 +521,6 @@ def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel) return interpreter -def _lowest_version_interpreter(interpreters): - """Given a list of interpreters, return the one with the lowest version.""" - lowest = interpreters[0] - for i in interpreters[1:]: - lowest = lowest if lowest < i else i - return lowest - - def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ @@ -538,7 +534,7 @@ def build_pex(args, options, resolver_option_builder): if options.interpreter_constraints: safe_constraints = options.interpreter_constraints.strip("'").strip('"') constraints = parse_interpreter_constraints(safe_constraints) - interpreters = list(matched_interpreters(interpreters, constraints, True)) + interpreters = list(matched_interpreters(interpreters, constraints, meet_all_constraints=True)) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) @@ -550,7 +546,7 @@ def build_pex(args, options, resolver_option_builder): # options.preamble_file is None preamble = None - interpreter = _lowest_version_interpreter(interpreters) + interpreter = lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index ac523b530..53df918db 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -3,30 +3,31 @@ # A library of functions for filtering Python interpreters based on compatibility constraints -def _matches(interpreter, filters, match_all=False): - if match_all: +def _matches(interpreter, filters, meet_all_constraints=False): + if meet_all_constraints: return all(interpreter.identity.matches(filt) for filt in filters) else: return any(interpreter.identity.matches(filt) for filt in filters) -def _matching(interpreters, filters, match_all=False): +def _matching(interpreters, filters, meet_all_constraints=False): for interpreter in interpreters: - if _matches(interpreter, filters, match_all): + if _matches(interpreter, filters, meet_all_constraints): yield interpreter -def matched_interpreters(interpreters, filters, match_all=False): +def matched_interpreters(interpreters, filters, meet_all_constraints=False): """Given some filters, yield any interpreter that matches at least one of them, or all of them - if match_all is set to True. + if meet_all_constraints is set to True. :param interpreters: a list of PythonInterpreter objects for filtering :param filters: A sequence of strings that constrain the interpreter compatibility for this pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class. - :param match_all: whether to match against all constraints. Defaults to matching one constraint. + :param meet_all_constraints: whether to match against all filters. + Defaults to matching interpreters that match at least one filter. """ - for match in _matching(interpreters, filters, match_all): + for match in _matching(interpreters, filters, meet_all_constraints): yield match @@ -45,3 +46,13 @@ def parse_interpreter_constraints(constraints_string): constraints_string.split(','))) else: return list(map(lambda x: x.strip(), constraints_string.split(','))) + + +def lowest_version_interpreter(interpreters): + """Given a list of interpreters, return the one with the lowest version.""" + if not interpreters: + return None + lowest = interpreters[0] + for i in interpreters[1:]: + lowest = lowest if lowest < i else i + return lowest diff --git a/tests/test_interpreter_constraints.py b/tests/test_interpreter_constraints.py new file mode 100644 index 000000000..bf1ad1880 --- /dev/null +++ b/tests/test_interpreter_constraints.py @@ -0,0 +1,18 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pex.interpreter_constraints import parse_interpreter_constraints + + +def test_parse_interpreter_constraints(): + example_input = '>=2.7, <3' + assert parse_interpreter_constraints(example_input) == ['>=2.7', '<3'] + + example_input = '>=2.7,<3' + assert parse_interpreter_constraints(example_input) == ['>=2.7', '<3'] + + example_input = 'CPython>=2.7,<3' + assert parse_interpreter_constraints(example_input) == ['CPython>=2.7', 'CPython<3'] + + example_input = 'CPython>=2.7, <3' + assert parse_interpreter_constraints(example_input) == ['CPython>=2.7', 'CPython<3'] From a6175dd83cd6846cd4f6da2e6aead4383115fd14 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 23 Oct 2017 16:04:10 -0700 Subject: [PATCH 04/43] Add pex python path to supported env variables; add interpreter constraints matching and search api for pex bootstrap re-exec; add associated unit + integration tests --- pex/pex_bootstrapper.py | 44 +++++++++++++++++----------- pex/variables.py | 12 ++++++++ tests/test_integration.py | 53 ++++++++++++++++++++++++++++++---- tests/test_pex_bootstrapper.py | 22 +++++++++++++- 4 files changed, 107 insertions(+), 24 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index c3510a84b..cdba6a071 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -5,6 +5,12 @@ import sys from .common import open_zip +from .interpreter import PythonInterpreter +from .interpreter_constraints import ( + lowest_version_interpreter, + matched_interpreters, + parse_interpreter_constraints +) __all__ = ('bootstrap_pex',) @@ -46,38 +52,42 @@ def get_pex_info(entry_point): raise ValueError('Invalid entry_point: %s' % entry_point) -def find_in_path(target_interpreter): - if os.path.exists(target_interpreter): - return target_interpreter +def _find_compatible_interpreter_in_pex_python_path(target_python_path, compatibility_constraints): + parsed_compatibility_constraints = parse_interpreter_constraints(compatibility_constraints) + try_binaries = [] + for binary in target_python_path.split(os.pathsep): + try_binaries.append(PythonInterpreter.from_binary(binary)) + compatible_interpreters = list(matched_interpreters( + try_binaries, parsed_compatibility_constraints, meet_all_constraints=True)) + return lowest_version_interpreter(compatible_interpreters) - for directory in os.getenv('PATH', '').split(os.pathsep): - try_path = os.path.join(directory, target_interpreter) - if os.path.exists(try_path): - return try_path - -def maybe_reexec_pex(): +def maybe_reexec_pex(compatibility_constraints=None): from .variables import ENV - if not ENV.PEX_PYTHON: + if not ENV.PEX_PYTHON_PATH or not compatibility_constraints: return from .common import die from .tracer import TRACER - target_python = ENV.PEX_PYTHON - target = find_in_path(target_python) + target_python_path = ENV.PEX_PYTHON_PATH + lowest_version_compatible_interpreter = _find_compatible_interpreter_in_pex_python_path( + target_python_path, compatibility_constraints) + target = lowest_version_compatible_interpreter.binary if not target: - die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) + die('Failed to find compatible interpreter in PEX_PYTHON_PATH for constraints: %s' + % compatibility_constraints) if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): - TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) - ENV.delete('PEX_PYTHON') - os.execve(target, [target_python] + sys.argv, ENV.copy()) + TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target) + ENV.delete('PEX_PYTHON_PATH') + os.execve(target, [target] + sys.argv, ENV.copy()) def bootstrap_pex(entry_point): from .finders import register_finders register_finders() - maybe_reexec_pex() + pex_info = get_pex_info(entry_point) + maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex pex.PEX(entry_point).execute() diff --git a/pex/variables.py b/pex/variables.py index 38208754d..cbc6a0983 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -231,6 +231,18 @@ def PEX_PYTHON(self): """ return self._get_string('PEX_PYTHON', default=None) + @property + def PEX_PYTHON_PATH(self): + """String + + A colon-seperated string containing paths to binaires of blessed Python interpreters + for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the + interpreter. + + Ex: "/path/to/python27:/path/to/python36" + """ + return self._get_string('PEX_PYTHON_PATH', default=None) + @property def PEX_ROOT(self): """Directory diff --git a/tests/test_integration.py b/tests/test_integration.py index df5943e69..452100593 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,8 +10,8 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller -from pex.pex_bootstrapper import read_pexinfo_from_zip -from pex.pex_info import PexInfo +from pex.interpreter import PythonInterpreter +from pex.pex_bootstrapper import get_pex_info from pex.testing import ( get_dep_dist_names_from_pex, run_pex_command, @@ -341,7 +341,7 @@ def test_interpreter_constraints_to_pex_info(): assert res.return_code == 102 else: res.assert_success() - pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + pex_info = get_pex_info(pex_out_path) assert '>=2.7,<3' == pex_info.interpreter_constraints # constraint with interpreter class @@ -353,7 +353,7 @@ def test_interpreter_constraints_to_pex_info(): assert res.return_code == 102 else: res.assert_success() - pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + pex_info = get_pex_info(pex_out_path) assert 'CPython>=2.7,<3' == pex_info.interpreter_constraints @@ -366,7 +366,7 @@ def test_interpreter_constraints_to_pex_info_py36(): '--interpreter-constraints=">=3"', '-o', pex_out_path]) res.assert_success() - pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + pex_info = get_pex_info(pex_out_path) assert '>=3' == pex_info.interpreter_constraints # constraint with interpreter class @@ -375,7 +375,7 @@ def test_interpreter_constraints_to_pex_info_py36(): '--interpreter-constraints="CPython>=3"', '-o', pex_out_path]) res.assert_success() - pex_info = PexInfo.from_json(read_pexinfo_from_zip(pex_out_path)) + pex_info = get_pex_info(pex_out_path) assert 'CPython>=3' == pex_info.interpreter_constraints @@ -398,3 +398,44 @@ def test_resolve_interpreter_with_constraints_option(): res.assert_success() else: assert res.return_code == 102 + + +@pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") +def test_interpreter_resolution_with_pex_python_path(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + # the line below will return the Python binaries installed on system, + # including binary of current tox env. If PPP is working correctly, the system binaries + # will be used to exec the pex instead of the executable of the current tox env. + interpreters = PythonInterpreter.all() + pi2 = list(filter(lambda i: '2.' in i.binary, interpreters)) + pi3 = list(filter(lambda i: '3.' in i.binary, interpreters)) + if not pi2 and not pi3: + print("Failed to find multiple python interpreters on system") + else: + with open(pexrc_path, 'w') as pexrc: + # set pex python path + pex_python_path = ':'.join([pi2[0].binary] + [pi3[0].binary]) + pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) + + # constraint to build pex cleanly; PPP + pex_bootstrapper.py + # will use these constraints to override sys.executable + interpreter_constraint = '>3' if sys.version_info[0] == 3 else '<3' + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints="%s"' % interpreter_constraint, + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + version = pi3[0].version if sys.version_info[0] == 3 else pi2[0].version + # check that the system python was used instead of tox env python, indicating success + if sys.version_info[0] == 3: + version_str = '.'.join([str(x)for x in version]) + bytes_version_str = version_str.encode() + assert bytes_version_str in stdout + else: + assert '.'.join([str(x)for x in version]) in stdout diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 27b38f7f9..594d6c4c4 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -6,7 +6,8 @@ from twitter.common.contextutil import temporary_dir from pex.common import open_zip -from pex.pex_bootstrapper import get_pex_info +from pex.interpreter import PythonInterpreter +from pex.pex_bootstrapper import _find_compatible_interpreter_in_pex_python_path, get_pex_info from pex.testing import write_simple_pex @@ -28,3 +29,22 @@ def test_get_pex_info(): # same when encoded assert pex_info.dump() == pex_info_2.dump() + + +def test_find_compatible_interpreter_in_python_path(): + interpreters = PythonInterpreter.all() + pi2 = list(filter(lambda x: '2.' in x.binary, interpreters)) + pi3 = list(filter(lambda x: '3.' in x.binary, interpreters)) + pex_python_path = ':'.join([pi2[0].binary] + [pi3[0].binary]) + + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') + assert interpreter.binary == pi2[0].binary + + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') + assert interpreter.binary == pi3[0].binary + + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<2') + assert interpreter is None + + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>4') + assert interpreter is None From 528d93ea3174362c589c8d20682867e9a4956ada Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 23 Oct 2017 17:51:32 -0700 Subject: [PATCH 05/43] Resolve interpreters for testing from .tox dir instead of system-specific installations --- tests/test_integration.py | 11 +++++++---- tests/test_pex_bootstrapper.py | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 452100593..4d4dd7cfb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -407,15 +407,18 @@ def test_interpreter_resolution_with_pex_python_path(): # the line below will return the Python binaries installed on system, # including binary of current tox env. If PPP is working correctly, the system binaries # will be used to exec the pex instead of the executable of the current tox env. - interpreters = PythonInterpreter.all() - pi2 = list(filter(lambda i: '2.' in i.binary, interpreters)) - pi3 = list(filter(lambda i: '3.' in i.binary, interpreters)) + root_dir = os.getcwd() + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), + PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] + pi2 = list(filter(lambda i: '2' in i.binary, interpreters)) + pi3 = list(filter(lambda i: '3' in i.binary, interpreters)) if not pi2 and not pi3: print("Failed to find multiple python interpreters on system") else: with open(pexrc_path, 'w') as pexrc: # set pex python path - pex_python_path = ':'.join([pi2[0].binary] + [pi3[0].binary]) + # for some reason pi2 from binary chops off 2.7 from the binary name so I add here + pex_python_path = ':'.join([pi2[0].binary + '2.7'] + [pi3[0].binary]) pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) # constraint to build pex cleanly; PPP + pex_bootstrapper.py diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 594d6c4c4..1c348114e 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -32,10 +32,13 @@ def test_get_pex_info(): def test_find_compatible_interpreter_in_python_path(): - interpreters = PythonInterpreter.all() - pi2 = list(filter(lambda x: '2.' in x.binary, interpreters)) - pi3 = list(filter(lambda x: '3.' in x.binary, interpreters)) - pex_python_path = ':'.join([pi2[0].binary] + [pi3[0].binary]) + root_dir = os.getcwd() + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), + PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] + pi2 = list(filter(lambda x: '2' in x.binary, interpreters)) + pi3 = list(filter(lambda x: '3' in x.binary, interpreters)) + # for some reason pi2 from binary chops off 2.7 from the binary name so I add here + pex_python_path = ':'.join([pi2[0].binary + '2.7'] + [pi3[0].binary]) interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') assert interpreter.binary == pi2[0].binary From 4fc655df0d6a4c8b93fc4943dad2bc85f0c49d74 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 24 Oct 2017 12:17:43 -0700 Subject: [PATCH 06/43] Refactor tests that use Python interpreters --- pex/interpreter_constraints.py | 20 ++++++++- tests/test_integration.py | 64 +++++++++++++-------------- tests/test_interpreter_constraints.py | 14 ++++++ tests/test_pex_bootstrapper.py | 31 ++++++++----- 4 files changed, 83 insertions(+), 46 deletions(-) diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 53df918db..0682085e6 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -3,6 +3,9 @@ # A library of functions for filtering Python interpreters based on compatibility constraints +from .interpreter import PythonIdentity + + def _matches(interpreter, filters, meet_all_constraints=False): if meet_all_constraints: return all(interpreter.identity.matches(filt) for filt in filters) @@ -16,6 +19,16 @@ def _matching(interpreters, filters, meet_all_constraints=False): yield interpreter +def check_requirements_are_well_formed(constraints): + # Check that the compatibility requirements are well-formed. + for req in constraints: + try: + PythonIdentity.parse_requirement(req) + except ValueError as e: + from .common import die + die("Compatibility requirements are not formatted properly: %s", str(e)) + + def matched_interpreters(interpreters, filters, meet_all_constraints=False): """Given some filters, yield any interpreter that matches at least one of them, or all of them if meet_all_constraints is set to True. @@ -41,11 +54,14 @@ def parse_interpreter_constraints(constraints_string): Example: 'CPython>=2.7,<3' Return: ['CPython>=2.7', 'CPython<3'] """ + if 'CPython' in constraints_string: - return list(map(lambda x: 'CPython' + x.strip() if not 'CPython' in x else x.strip(), + ret = list(map(lambda x: 'CPython' + x.strip() if 'CPython' not in x else x.strip(), constraints_string.split(','))) else: - return list(map(lambda x: x.strip(), constraints_string.split(','))) + ret = list(map(lambda x: x.strip(), constraints_string.split(','))) + check_requirements_are_well_formed(ret) + return ret def lowest_version_interpreter(interpreters): diff --git a/tests/test_integration.py b/tests/test_integration.py index 4d4dd7cfb..55dfdc42b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -408,37 +408,37 @@ def test_interpreter_resolution_with_pex_python_path(): # including binary of current tox env. If PPP is working correctly, the system binaries # will be used to exec the pex instead of the executable of the current tox env. root_dir = os.getcwd() - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), - PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] - pi2 = list(filter(lambda i: '2' in i.binary, interpreters)) - pi3 = list(filter(lambda i: '3' in i.binary, interpreters)) - if not pi2 and not pi3: - print("Failed to find multiple python interpreters on system") + if sys.version_info[0] == 3: + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py36-requests/bin/python3.6'), + PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] else: - with open(pexrc_path, 'w') as pexrc: - # set pex python path - # for some reason pi2 from binary chops off 2.7 from the binary name so I add here - pex_python_path = ':'.join([pi2[0].binary + '2.7'] + [pi3[0].binary]) - pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) - - # constraint to build pex cleanly; PPP + pex_bootstrapper.py - # will use these constraints to override sys.executable - interpreter_constraint = '>3' if sys.version_info[0] == 3 else '<3' - - pex_out_path = os.path.join(td, 'pex.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraints="%s"' % interpreter_constraint, - '-o', pex_out_path]) - res.assert_success() + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27-requests/bin/python2.7'), + PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7')] - stdin_payload = b'import sys; sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) - assert rc == 0 - version = pi3[0].version if sys.version_info[0] == 3 else pi2[0].version - # check that the system python was used instead of tox env python, indicating success - if sys.version_info[0] == 3: - version_str = '.'.join([str(x)for x in version]) - bytes_version_str = version_str.encode() - assert bytes_version_str in stdout - else: - assert '.'.join([str(x)for x in version]) in stdout + with open(pexrc_path, 'w') as pexrc: + # set pex python path + pex_python_path = ':'.join([interpreters[0].binary] + [interpreters[1].binary]) + pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) + + # constraint to build pex cleanly; PPP + pex_bootstrapper.py + # will use these constraints to override sys.executable + interpreter_constraint = '>3' if sys.version_info[0] == 3 else '<3' + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraints="%s"' % interpreter_constraint, + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + + version = interpreters[1].version + # check that the pex python path python was used instead of tox env python, indicating success + if sys.version_info[0] == 3: + version_str = '.'.join([str(x)for x in version]) + bytes_version_str = version_str.encode() + assert bytes_version_str in stdout + else: + assert '.'.join([str(x)for x in version]) in stdout diff --git a/tests/test_interpreter_constraints.py b/tests/test_interpreter_constraints.py index bf1ad1880..2a3aa632b 100644 --- a/tests/test_interpreter_constraints.py +++ b/tests/test_interpreter_constraints.py @@ -1,6 +1,8 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import pytest + from pex.interpreter_constraints import parse_interpreter_constraints @@ -16,3 +18,15 @@ def test_parse_interpreter_constraints(): example_input = 'CPython>=2.7, <3' assert parse_interpreter_constraints(example_input) == ['CPython>=2.7', 'CPython<3'] + + example_input = 'CPython==3.6' + assert parse_interpreter_constraints(example_input) == ['CPython==3.6'] + + with pytest.raises(SystemExit) as e: + example_input = 'CPython>=?.2.7,=<3>' + parse_interpreter_constraints(example_input) + assert 'Unknown requirement string' in str(e.info) + + example_input = '==2,2.7><,3_9' + parse_interpreter_constraints(example_input) + assert 'Unknown requirement string' in str(e.info) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 1c348114e..8822d7c2d 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -2,7 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys +import pytest from twitter.common.contextutil import temporary_dir from pex.common import open_zip @@ -31,20 +33,25 @@ def test_get_pex_info(): assert pex_info.dump() == pex_info_2.dump() +@pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_find_compatible_interpreter_in_python_path(): root_dir = os.getcwd() - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), - PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] - pi2 = list(filter(lambda x: '2' in x.binary, interpreters)) - pi3 = list(filter(lambda x: '3' in x.binary, interpreters)) - # for some reason pi2 from binary chops off 2.7 from the binary name so I add here - pex_python_path = ':'.join([pi2[0].binary + '2.7'] + [pi3[0].binary]) - - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') - assert interpreter.binary == pi2[0].binary - - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') - assert interpreter.binary == pi3[0].binary + if sys.version_info[0] == 3: + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6'), + PythonInterpreter.from_binary(root_dir + '/.tox/py36-requests/bin/python3.6')] + else: + interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), + PythonInterpreter.from_binary(root_dir + '/.tox/py27-requests/bin/python2.7')] + + pex_python_path = ':'.join([interpreters[0].binary] + [interpreters[1].binary]) + + if sys.version_info[0] == 3: + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') + # the returned interpreter will the rightmost interpreter in PPP if all versions are the same + assert interpreter.binary == interpreters[1].binary + else: + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') + assert interpreter.binary == interpreters[1].binary interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<2') assert interpreter is None From ca8a9c684b84a003fe2a8d0a1f9ccf63462cacfd Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 24 Oct 2017 16:44:22 -0700 Subject: [PATCH 07/43] Add mock patching for pex python path search api test --- pex/pex_bootstrapper.py | 6 ++- tests/test_pex_bootstrapper.py | 71 +++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index cdba6a071..31a9533d4 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -52,11 +52,15 @@ def get_pex_info(entry_point): raise ValueError('Invalid entry_point: %s' % entry_point) +def _get_python_interpreter(binary): + return PythonInterpreter.from_binary(binary) + + def _find_compatible_interpreter_in_pex_python_path(target_python_path, compatibility_constraints): parsed_compatibility_constraints = parse_interpreter_constraints(compatibility_constraints) try_binaries = [] for binary in target_python_path.split(os.pathsep): - try_binaries.append(PythonInterpreter.from_binary(binary)) + try_binaries.append(_get_python_interpreter(binary)) compatible_interpreters = list(matched_interpreters( try_binaries, parsed_compatibility_constraints, meet_all_constraints=True)) return lowest_version_interpreter(compatible_interpreters) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 8822d7c2d..00ac07ca7 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -2,16 +2,59 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import sys import pytest from twitter.common.contextutil import temporary_dir from pex.common import open_zip -from pex.interpreter import PythonInterpreter from pex.pex_bootstrapper import _find_compatible_interpreter_in_pex_python_path, get_pex_info from pex.testing import write_simple_pex +try: + import mock +except ImportError: + import unittest.mock as mock + + + + +@pytest.fixture +def py27_interpreter(): + mock_interpreter = mock.MagicMock() + mock_interpreter.binary = '/path/to/python2.7' + mock_interpreter.version = (2, 7, 10) + mock_interpreter.__lt__ = lambda x, y: x.version < y.version + return mock_interpreter + + +@pytest.fixture +def py36_interpreter(): + mock_interpreter = mock.MagicMock() + mock_interpreter.binary = '/path/to/python3.6' + mock_interpreter.version = (3, 6, 3) + mock_interpreter.__lt__ = lambda x, y: x.version < y.version + return mock_interpreter + + +def mock_get_python_interpreter(binary): + """Patch function for resolving PythonInterpreter mock objects from Pex Python Path""" + if '3' in binary: + return py36_interpreter() + elif '2' in binary: + return py27_interpreter() + + +def mock_matches(interpreter, filters, meet_all_constraints): + """Patch function for determining if the supplied interpreter complies with the filters""" + if '>3' in filters: + return True if interpreter.version > (3, 0, 0) else False + elif '<3' in filters: + return True if interpreter.version < (3, 0, 0) else False + elif '>=2.7' in filters: + return True if interpreter.version > (2, 7, 0) else False + else: + return False + def test_get_pex_info(): with temporary_dir() as td: @@ -33,25 +76,17 @@ def test_get_pex_info(): assert pex_info.dump() == pex_info_2.dump() +@mock.patch('pex.interpreter_constraints._matches', side_effect=mock_matches) +@mock.patch('pex.pex_bootstrapper._get_python_interpreter', side_effect=mock_get_python_interpreter) @pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") -def test_find_compatible_interpreter_in_python_path(): - root_dir = os.getcwd() - if sys.version_info[0] == 3: - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6'), - PythonInterpreter.from_binary(root_dir + '/.tox/py36-requests/bin/python3.6')] - else: - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7'), - PythonInterpreter.from_binary(root_dir + '/.tox/py27-requests/bin/python2.7')] +def test_find_compatible_interpreter_in_python_path(mock_get_python_interpreter, mock_matches): + pex_python_path = ':'.join(['/path/to/python2.7', '/path/to/python3.6']) - pex_python_path = ':'.join([interpreters[0].binary] + [interpreters[1].binary]) + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') + assert interpreter.binary == '/path/to/python3.6' - if sys.version_info[0] == 3: - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') - # the returned interpreter will the rightmost interpreter in PPP if all versions are the same - assert interpreter.binary == interpreters[1].binary - else: - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') - assert interpreter.binary == interpreters[1].binary + interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') + assert interpreter.binary == '/path/to/python2.7' interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<2') assert interpreter is None From 02fa8c5174ccfd3fc35368151a21c46e1c162831 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:00:02 -0700 Subject: [PATCH 08/43] Add interpreter bootstrapping as test helper method, make all interpreter selection tests green; choose highest compatible interpreter in pex python path instead of lowest. --- .gitignore | 1 + .travis.yml | 4 ++++ pex/interpreter_constraints.py | 10 ++++++++++ pex/pex_bootstrapper.py | 4 ++-- pex/testing.py | 15 +++++++++++++++ tests/test_integration.py | 33 ++++++++++----------------------- tests/test_pex_bootstrapper.py | 3 --- 7 files changed, 42 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 1226a163e..605887ffd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /.idea /.coverage* /htmlcov +/.pyenv diff --git a/.travis.yml b/.travis.yml index 302983e0f..ab60930b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ dist: precise # TRAVIS_PYTHON_VERSION +cache: + directories: + - .pyenv/ + matrix: include: - language: python diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 0682085e6..01a2c3f75 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -72,3 +72,13 @@ def lowest_version_interpreter(interpreters): for i in interpreters[1:]: lowest = lowest if lowest < i else i return lowest + + +def highest_version_interpreter(interpreters): + """Given a list of interpreters, return the one with the highest version.""" + if not interpreters: + return None + highest = interpreters[0] + for i in interpreters[1:]: + highest = i if highest < i else highest + return highest diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 31a9533d4..701f773a5 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -7,7 +7,7 @@ from .common import open_zip from .interpreter import PythonInterpreter from .interpreter_constraints import ( - lowest_version_interpreter, + highest_version_interpreter, matched_interpreters, parse_interpreter_constraints ) @@ -63,7 +63,7 @@ def _find_compatible_interpreter_in_pex_python_path(target_python_path, compatib try_binaries.append(_get_python_interpreter(binary)) compatible_interpreters = list(matched_interpreters( try_binaries, parsed_compatibility_constraints, meet_all_constraints=True)) - return lowest_version_interpreter(compatible_interpreters) + return highest_version_interpreter(compatible_interpreters) def maybe_reexec_pex(compatibility_constraints=None): diff --git a/pex/testing.py b/pex/testing.py index 215071eb4..19ad13f34 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -4,6 +4,7 @@ import contextlib import os import random +import subprocess import sys import tempfile from collections import namedtuple @@ -277,3 +278,17 @@ def combine_pex_coverage(coverage_file_iter): combined.write() return combined.filename + + +def bootstrap_python_installer(): + install_location = os.getcwd() + '/.pyenv' + if not os.path.exists(install_location): + subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + + +def ensure_python_interpreter(version): + bootstrap_python_installer() + install_location = os.getcwd() + '/.pyenv/versions/' + version + if not os.path.exists(install_location): + os.environ['PYENV_ROOT'] = os.getcwd() + '/.pyenv' + subprocess.call([os.getcwd() + '/.pyenv/bin/pyenv', 'install', version]) diff --git a/tests/test_integration.py b/tests/test_integration.py index 55dfdc42b..005fe2b00 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,9 +10,9 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller -from pex.interpreter import PythonInterpreter from pex.pex_bootstrapper import get_pex_info from pex.testing import ( + ensure_python_interpreter, get_dep_dist_names_from_pex, run_pex_command, run_simple_pex, @@ -402,27 +402,19 @@ def test_resolve_interpreter_with_constraints_option(): @pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_interpreter_resolution_with_pex_python_path(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') - # the line below will return the Python binaries installed on system, - # including binary of current tox env. If PPP is working correctly, the system binaries - # will be used to exec the pex instead of the executable of the current tox env. - root_dir = os.getcwd() - if sys.version_info[0] == 3: - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py36-requests/bin/python3.6'), - PythonInterpreter.from_binary(root_dir + '/.tox/py36/bin/python3.6')] - else: - interpreters = [PythonInterpreter.from_binary(root_dir + '/.tox/py27-requests/bin/python2.7'), - PythonInterpreter.from_binary(root_dir + '/.tox/py27/bin/python2.7')] - with open(pexrc_path, 'w') as pexrc: # set pex python path - pex_python_path = ':'.join([interpreters[0].binary] + [interpreters[1].binary]) + pex_python_path = ':'.join([os.getcwd() + '/.pyenv/versions/2.7.10/bin/python', + os.getcwd() + '/.pyenv/versions/3.6.3/bin/python']) pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) # constraint to build pex cleanly; PPP + pex_bootstrapper.py - # will use these constraints to override sys.executable - interpreter_constraint = '>3' if sys.version_info[0] == 3 else '<3' + # will use these constraints to override sys.executable on pex re-exec + interpreter_constraint = '>3' if sys.version_info[0] == 3 else '>=2.7,<3' pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', @@ -430,15 +422,10 @@ def test_interpreter_resolution_with_pex_python_path(): '-o', pex_out_path]) res.assert_success() - stdin_payload = b'import sys; sys.exit(0)' + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 - - version = interpreters[1].version - # check that the pex python path python was used instead of tox env python, indicating success if sys.version_info[0] == 3: - version_str = '.'.join([str(x)for x in version]) - bytes_version_str = version_str.encode() - assert bytes_version_str in stdout + assert str(pex_python_path.split(':')[1]).encode() in stdout else: - assert '.'.join([str(x)for x in version]) in stdout + assert str(pex_python_path.split(':')[0]).encode() in stdout diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 00ac07ca7..83179517e 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -16,8 +16,6 @@ import unittest.mock as mock - - @pytest.fixture def py27_interpreter(): mock_interpreter = mock.MagicMock() @@ -78,7 +76,6 @@ def test_get_pex_info(): @mock.patch('pex.interpreter_constraints._matches', side_effect=mock_matches) @mock.patch('pex.pex_bootstrapper._get_python_interpreter', side_effect=mock_get_python_interpreter) -@pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_find_compatible_interpreter_in_python_path(mock_get_python_interpreter, mock_matches): pex_python_path = ':'.join(['/path/to/python2.7', '/path/to/python3.6']) From 64c2b0c12e653384a6fd46495dc4d76f42ac4667 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:21:41 -0700 Subject: [PATCH 09/43] Travis CI debugging --- pex/testing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index 19ad13f34..75c4f441b 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -282,8 +282,9 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): install_location = os.getcwd() + '/.pyenv' - if not os.path.exists(install_location): - subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + #if not os.path.exists(install_location): + subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + print(os.listdir(install_location)) def ensure_python_interpreter(version): From 3408fb35110bb2574bc0e650d187ab40694899c5 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:30:20 -0700 Subject: [PATCH 10/43] Add pytest.mark modification to skip test in CI --- pex/testing.py | 5 ++--- tests/test_integration.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index 75c4f441b..19ad13f34 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -282,9 +282,8 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): install_location = os.getcwd() + '/.pyenv' - #if not os.path.exists(install_location): - subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) - print(os.listdir(install_location)) + if not os.path.exists(install_location): + subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) def ensure_python_interpreter(version): diff --git a/tests/test_integration.py b/tests/test_integration.py index 005fe2b00..d94e271fe 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -400,6 +400,7 @@ def test_resolve_interpreter_with_constraints_option(): assert res.return_code == 102 +@pytest.mark.skipif("sys.platform != 'darwin'") @pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_interpreter_resolution_with_pex_python_path(): ensure_python_interpreter('2.7.10') From a1c4fedc21cb941ce17e7f72181004cfec0f3780 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:42:58 -0700 Subject: [PATCH 11/43] Travis CI debug --- pex/testing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pex/testing.py b/pex/testing.py index 19ad13f34..ffa7d7f3e 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -283,7 +283,11 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): install_location = os.getcwd() + '/.pyenv' if not os.path.exists(install_location): - subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + #subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + p = subprocess.Popen(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location], + stdout=subprocess.PIPE) + print p.communicate() + raise ValueError(p.communicate()) def ensure_python_interpreter(version): From cef5bf55f86756e25028f80be2f5c1fc1bc90bf5 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:42:58 -0700 Subject: [PATCH 12/43] Travis CI debug --- pex/testing.py | 6 +++++- tests/test_integration.py | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index 19ad13f34..31e4d63e4 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -283,7 +283,11 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): install_location = os.getcwd() + '/.pyenv' if not os.path.exists(install_location): - subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + #subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) + p = subprocess.Popen(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location], + stdout=subprocess.PIPE) + print(p.communicate()) + raise ValueError(p.communicate()) def ensure_python_interpreter(version): diff --git a/tests/test_integration.py b/tests/test_integration.py index d94e271fe..005fe2b00 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -400,7 +400,6 @@ def test_resolve_interpreter_with_constraints_option(): assert res.return_code == 102 -@pytest.mark.skipif("sys.platform != 'darwin'") @pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_interpreter_resolution_with_pex_python_path(): ensure_python_interpreter('2.7.10') From 0877ad0cdf5298de3a60484466679a08f43dd0d7 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 17:58:16 -0700 Subject: [PATCH 13/43] Travis CI debug 2 --- pex/testing.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index 31e4d63e4..5a5324d92 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import contextlib +import logging import os import random import subprocess @@ -19,6 +20,8 @@ from .util import DistributionHelper, named_temporary_file +log = logging.getLogger(__name__) + @contextlib.contextmanager def temporary_dir(): td = tempfile.mkdtemp() @@ -283,11 +286,10 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): install_location = os.getcwd() + '/.pyenv' if not os.path.exists(install_location): - #subprocess.call(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location]) p = subprocess.Popen(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location], stdout=subprocess.PIPE) - print(p.communicate()) - raise ValueError(p.communicate()) + log.info("LOGGING SUBPROCESS: %s" % p.communicate()) + def ensure_python_interpreter(version): From 762e9d3e91f0dd5aeacc07cfda417db11cd0a147 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 25 Oct 2017 18:12:00 -0700 Subject: [PATCH 14/43] Travis CI test 3 --- .travis.yml | 3 +++ pex/testing.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ab60930b6..0a72e0ec2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ cache: directories: - .pyenv/ +install: + - git clone https://github.com/pyenv/pyenv.git ./.pyenv + matrix: include: - language: python diff --git a/pex/testing.py b/pex/testing.py index 5a5324d92..c50fa4334 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -291,7 +291,6 @@ def bootstrap_python_installer(): log.info("LOGGING SUBPROCESS: %s" % p.communicate()) - def ensure_python_interpreter(version): bootstrap_python_installer() install_location = os.getcwd() + '/.pyenv/versions/' + version From ebf830a3eeefd66a7e8ea5594970bd711f88774c Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Thu, 26 Oct 2017 10:33:23 -0700 Subject: [PATCH 15/43] Remove logging --- pex/testing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index c50fa4334..c79dcb0be 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import contextlib -import logging import os import random import subprocess @@ -20,7 +19,6 @@ from .util import DistributionHelper, named_temporary_file -log = logging.getLogger(__name__) @contextlib.contextmanager def temporary_dir(): @@ -288,8 +286,7 @@ def bootstrap_python_installer(): if not os.path.exists(install_location): p = subprocess.Popen(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location], stdout=subprocess.PIPE) - log.info("LOGGING SUBPROCESS: %s" % p.communicate()) - + print(p.communicate()) def ensure_python_interpreter(version): bootstrap_python_installer() From 233918eabc391fed64f9d631d92f5ad49b0d458e Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 8 Nov 2017 12:08:41 -0800 Subject: [PATCH 16/43] Modify pex bootstrapper to repsect pex_python_path environment variable, but maintain support for pex_python if still set. Add relevant integration tests to test each of the test cases that deal with interpreter constraints. --- .gitignore | 2 +- .travis.yml | 5 +- pex/bin/pex.py | 35 ++--- pex/interpreter_constraints.py | 42 +----- pex/pex_bootstrapper.py | 123 +++++++++++++---- pex/pex_builder.py | 9 ++ pex/pex_info.py | 12 +- pex/testing.py | 24 ++-- pex/variables.py | 21 +++ tests/test_integration.py | 190 +++++++++++++++++++++++--- tests/test_interpreter_constraints.py | 32 ----- tests/test_pex_bootstrapper.py | 98 ++++++------- 12 files changed, 379 insertions(+), 214 deletions(-) delete mode 100644 tests/test_interpreter_constraints.py diff --git a/.gitignore b/.gitignore index 605887ffd..b05ce0476 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ /.idea /.coverage* /htmlcov -/.pyenv +/.pyenv_test diff --git a/.travis.yml b/.travis.yml index 0a72e0ec2..01ecd2aa3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,7 @@ dist: precise cache: directories: - - .pyenv/ - -install: - - git clone https://github.com/pyenv/pyenv.git ./.pyenv + - .pyenv_test matrix: include: diff --git a/pex/bin/pex.py b/pex/bin/pex.py index d32504eee..d4a26ae20 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,11 +23,7 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter -from pex.interpreter_constraints import ( - lowest_version_interpreter, - matched_interpreters, - parse_interpreter_constraints -) +from pex.interpreter_constraints import check_requirements_are_well_formed, matched_interpreters from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX @@ -295,13 +291,15 @@ def configure_clp_pex_environment(parser): 'Default: Use current interpreter.') group.add_option( - '--interpreter-constraints', - dest='interpreter_constraints', - default='', + '--interpreter-constraint', + dest='interpreter_constraint', + default=[], type='str', - help='A comma-seperated list of constraints that determine the interpreter compatibility for ' - 'this pex, using the Requirement-style format, e.g. "CPython>=3", or just ">=2.7,<3" ' - 'for requirements agnostic to interpreter class.') + action='append', + help='A constraint that determines the interpreter compatibility for ' + 'this pex, using the Requirement-style format, e.g. "CPython>=3", or ">=2.7" ' + 'for requirements agnostic to interpreter class. This option can be passed multiple ' + 'times.') group.add_option( '--python-shebang', @@ -531,9 +529,10 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] - if options.interpreter_constraints: - safe_constraints = options.interpreter_constraints.strip("'").strip('"') - constraints = parse_interpreter_constraints(safe_constraints) + if options.interpreter_constraint: + constraints = options.interpreter_constraint + # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted + check_requirements_are_well_formed(constraints) interpreters = list(matched_interpreters(interpreters, constraints, meet_all_constraints=True)) if not interpreters: @@ -546,7 +545,8 @@ def build_pex(args, options, resolver_option_builder): # options.preamble_file is None preamble = None - interpreter = lowest_version_interpreter(interpreters) + # TODO: make whether to take min or max version interpreter configurable + interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info @@ -555,8 +555,9 @@ def build_pex(args, options, resolver_option_builder): pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path - if options.interpreter_constraints: - pex_info.interpreter_constraints = safe_constraints + if options.interpreter_constraint: + for ic in constraints: + pex_builder.add_interpreter_constraint(ic) resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 01a2c3f75..33814cb27 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -26,7 +26,7 @@ def check_requirements_are_well_formed(constraints): PythonIdentity.parse_requirement(req) except ValueError as e: from .common import die - die("Compatibility requirements are not formatted properly: %s", str(e)) + die("Compatibility requirements are not formatted properly: %s" % str(e)) def matched_interpreters(interpreters, filters, meet_all_constraints=False): @@ -42,43 +42,3 @@ def matched_interpreters(interpreters, filters, meet_all_constraints=False): """ for match in _matching(interpreters, filters, meet_all_constraints): yield match - - -def parse_interpreter_constraints(constraints_string): - """Given a single string defining interpreter constraints, separate them into a list of - individual constraint items for PythonIdentity to consume. - - Example: '>=2.7, <3' - Return: ['>=2.7', '<3'] - - Example: 'CPython>=2.7,<3' - Return: ['CPython>=2.7', 'CPython<3'] - """ - - if 'CPython' in constraints_string: - ret = list(map(lambda x: 'CPython' + x.strip() if 'CPython' not in x else x.strip(), - constraints_string.split(','))) - else: - ret = list(map(lambda x: x.strip(), constraints_string.split(','))) - check_requirements_are_well_formed(ret) - return ret - - -def lowest_version_interpreter(interpreters): - """Given a list of interpreters, return the one with the lowest version.""" - if not interpreters: - return None - lowest = interpreters[0] - for i in interpreters[1:]: - lowest = lowest if lowest < i else i - return lowest - - -def highest_version_interpreter(interpreters): - """Given a list of interpreters, return the one with the highest version.""" - if not interpreters: - return None - highest = interpreters[0] - for i in interpreters[1:]: - highest = i if highest < i else highest - return highest diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 701f773a5..accaeaac6 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -4,13 +4,12 @@ import os import sys -from .common import open_zip +from .common import die, open_zip +from .executor import Executor from .interpreter import PythonInterpreter -from .interpreter_constraints import ( - highest_version_interpreter, - matched_interpreters, - parse_interpreter_constraints -) +from .interpreter_constraints import matched_interpreters +from .tracer import TRACER +from .variables import ENV __all__ = ('bootstrap_pex',) @@ -52,46 +51,110 @@ def get_pex_info(entry_point): raise ValueError('Invalid entry_point: %s' % entry_point) -def _get_python_interpreter(binary): - return PythonInterpreter.from_binary(binary) +def find_in_path(target_interpreter): + if os.path.exists(target_interpreter): + return target_interpreter + + for directory in os.getenv('PATH', '').split(os.pathsep): + try_path = os.path.join(directory, target_interpreter) + if os.path.exists(try_path): + return try_path -def _find_compatible_interpreter_in_pex_python_path(target_python_path, compatibility_constraints): - parsed_compatibility_constraints = parse_interpreter_constraints(compatibility_constraints) - try_binaries = [] - for binary in target_python_path.split(os.pathsep): - try_binaries.append(_get_python_interpreter(binary)) +def _get_python_interpreter(binary): + try: + return PythonInterpreter.from_binary(binary) + # from_binary attempts to execute the interpreter + except Executor.ExecutionError: + return None + + +def _find_compatible_interpreters(pex_python_path, compatibility_constraints): + """Find all compatible interpreters on the system within the supplied constraints and use + PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. + """ + if pex_python_path: + interpreters = [] + for binary in pex_python_path.split(os.pathsep): + pi = _get_python_interpreter(binary) + if pi: + interpreters.append(pi) + if not interpreters: + die('No interpreters from PEX_PYTHON_PATH can be found on the system. Exiting.') + else: + # All qualifying interpreters found in $PATH + interpreters = PythonInterpreter.all() + compatible_interpreters = list(matched_interpreters( - try_binaries, parsed_compatibility_constraints, meet_all_constraints=True)) - return highest_version_interpreter(compatible_interpreters) + interpreters, compatibility_constraints, meet_all_constraints=True)) + return compatible_interpreters if compatible_interpreters else None -def maybe_reexec_pex(compatibility_constraints=None): - from .variables import ENV - if not ENV.PEX_PYTHON_PATH or not compatibility_constraints: - return +def _handle_pex_python(target_python, compatibility_constraints): + """Re-exec using the PEX_PYTHON interpreter""" + target = find_in_path(target_python) + if not target: + die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) + elif compatibility_constraints: + pi = PythonInterpreter.from_binary(target) + if not all(pi.identity.matches(constraint) for constraint in compatibility_constraints): + die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' + 'interpreter constraints: %s' % (target, str(compatibility_constraints))) + if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): + TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) + ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True + os.execve(target, [target_python] + sys.argv, ENV.copy()) + - from .common import die - from .tracer import TRACER +def _handle_general_interpreter_selection(pex_python_path, compatibility_constraints): + """Handle selection in the case that PEX_PYTHON_PATH is set or interpreter compatibility + constraints are specified. + """ + compatible_interpreters = _find_compatible_interpreters( + pex_python_path, compatibility_constraints) - target_python_path = ENV.PEX_PYTHON_PATH - lowest_version_compatible_interpreter = _find_compatible_interpreter_in_pex_python_path( - target_python_path, compatibility_constraints) - target = lowest_version_compatible_interpreter.binary + target = None + if compatible_interpreters: + target = min(compatible_interpreters).binary if not target: - die('Failed to find compatible interpreter in PEX_PYTHON_PATH for constraints: %s' - % compatibility_constraints) + die('Failed to find compatible interpreter for constraints: %s' + % str(compatibility_constraints)) if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): - TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target) - ENV.delete('PEX_PYTHON_PATH') + if pex_python_path: + TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target) + else: + TRACER.log('Re-exec to interpreter %s that matches constraints: %s' % (target, + str(compatibility_constraints))) + ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True os.execve(target, [target] + sys.argv, ENV.copy()) +def maybe_reexec_pex(compatibility_constraints=None): + if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC: + return + if ENV.PEX_PYTHON and ENV.PEX_PYTHON_PATH: + # both vars are defined, fall through to PEX_PYTHON_PATH resolution (give precedence to PPP) + pass + elif ENV.PEX_PYTHON: + # preserve PEX_PYTHON re-exec for backwards compatibility + _handle_pex_python(ENV.PEX_PYTHON, compatibility_constraints) + + if not compatibility_constraints: + # if no compatibility constraints are specified, we want to match against + # the lowest-versioned interpreter in PEX_PYTHON_PATH if it is set + if not ENV.PEX_PYTHON_PATH: + # no PEX_PYTHON_PATH, PEX_PYTHON, or interpreter constraints, continue as normal + return + + _handle_general_interpreter_selection(ENV.PEX_PYTHON_PATH, compatibility_constraints) + + def bootstrap_pex(entry_point): from .finders import register_finders register_finders() pex_info = get_pex_info(entry_point) - maybe_reexec_pex(pex_info.interpreter_constraints) + with TRACER.timed('Bootstrapping pex with maybe_reexec_pex', V=2): + maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex pex.PEX(entry_point).execute() diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 36b4056d5..e98e74937 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -166,6 +166,15 @@ def add_requirement(self, req): self._ensure_unfrozen('Adding a requirement') self._pex_info.add_requirement(req) + def add_interpreter_constraint(self, ic): + """Add an interpreter constraint to the PEX environment. + + :param ic: A version constraint on the interpreter used to build and run this PEX environment. + + """ + self._ensure_unfrozen('Adding an interpreter constraint') + self._pex_info.add_interpreter_constraint(ic) + def set_executable(self, filename, env_filename=None): """Set the executable for this environment. diff --git a/pex/pex_info.py b/pex/pex_info.py index 1d11bcb0c..f1ce90524 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -123,6 +123,7 @@ def __init__(self, info=None): '%s of type %s' % (info, type(info))) self._pex_info = info or {} self._distributions = self._pex_info.get('distributions', {}) + self._interpreter_constraints = self._pex_info.get('interpreter_constraints', []) requirements = self._pex_info.get('requirements', []) if not isinstance(requirements, (list, tuple)): raise ValueError('Expected requirements to be a list, got %s' % type(requirements)) @@ -197,18 +198,17 @@ def inherit_path(self, value): @property def interpreter_constraints(self): - """A comma-seperated list of constraints that determine the interpreter compatibility for this + """A list of constraints that determine the interpreter compatibility for this pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'`` for requirements agnostic to interpreter class. This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH for a list of compatible interpreters. """ - return self._pex_info.get('interpreter_constraints', False) + return self._interpreter_constraints - @interpreter_constraints.setter - def interpreter_constraints(self, value): - self._pex_info['interpreter_constraints'] = value + def add_interpreter_constraint(self, value): + self._interpreter_constraints.append(str(value)) @property def ignore_errors(self): @@ -289,11 +289,13 @@ def update(self, other): raise TypeError('Cannot merge a %r with PexInfo' % type(other)) self._pex_info.update(other._pex_info) self._distributions.update(other.distributions) + self._interpreter_constraints.extend(other.interpreter_constraints) self._requirements.update(other.requirements) def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) + pex_info_copy['interpreter_constraints'] = self._interpreter_constraints pex_info_copy['distributions'] = self._distributions.copy() return json.dumps(pex_info_copy, **kwargs) diff --git a/pex/testing.py b/pex/testing.py index c79dcb0be..65098b461 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -19,7 +19,6 @@ from .util import DistributionHelper, named_temporary_file - @contextlib.contextmanager def temporary_dir(): td = tempfile.mkdtemp() @@ -282,15 +281,22 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): - install_location = os.getcwd() + '/.pyenv' - if not os.path.exists(install_location): - p = subprocess.Popen(["git", "clone", 'https://github.com/pyenv/pyenv.git', install_location], - stdout=subprocess.PIPE) - print(p.communicate()) + install_location = os.getcwd() + '/.pyenv_test' + if not os.path.exists(install_location) or not os.path.exists(install_location + '/bin/pyenv'): + for _ in range(3): + try: + subprocess.call(['git', 'clone', 'https://github.com/pyenv/pyenv.git', install_location]) + except StandardError: + continue + else: + break + else: + raise RuntimeError("Helper method could not clone pyenv from git") + def ensure_python_interpreter(version): bootstrap_python_installer() - install_location = os.getcwd() + '/.pyenv/versions/' + version + install_location = os.getcwd() + '/.pyenv_test/versions/' + version if not os.path.exists(install_location): - os.environ['PYENV_ROOT'] = os.getcwd() + '/.pyenv' - subprocess.call([os.getcwd() + '/.pyenv/bin/pyenv', 'install', version]) + os.environ['PYENV_ROOT'] = os.getcwd() + '/.pyenv_test' + subprocess.call([os.getcwd() + '/.pyenv_test/bin/pyenv', 'install', version]) diff --git a/pex/variables.py b/pex/variables.py index cbc6a0983..44fb4cda5 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -312,6 +312,27 @@ def PEX_IGNORE_RCFILES(self): """ return self._get_bool('PEX_IGNORE_RCFILES', default=False) + @property + def SHOULD_EXIT_BOOTSTRAP_REEXEC(self): + """Boolean + + Whether to re-exec in maybe_reexec_pex function of pex_bootstrapper.py. Default: false. + This is neccesary because that function relies on checking against variables present in the + ENV to determine whether to re-exec or return to current execution. When we introduced + interpreter constraints to a pex, these constraints can influence interpreter selection without + the need for a PEX_PYTHON or PEX_PYTHON_PATH ENV variable set. Since re-exec previously checked + for PEX_PYTHON or PEX_PYTHON_PATH but not constraints, this would result in a loop with no + stopping criteria. Setting SHOULD_EXIT_BOOTSTRAP_REEXEC will let the runtime know to break out + of a second execution of maybe_reexec_pex if neither PEX_PYTHON or PEX_PYTHON_PATH are set but + interpreter constraints are specified. + """ + value = self._environ.get('SHOULD_EXIT_REEXEC', 'False') + return False if value == 'False' else True + + @SHOULD_EXIT_BOOTSTRAP_REEXEC.setter + def SHOULD_EXIT_BOOTSTRAP_REEXEC(self, value): + self._environ['SHOULD_EXIT_REEXEC'] = str(value) + # Global singleton environment ENV = Variables() diff --git a/tests/test_integration.py b/tests/test_integration.py index 005fe2b00..bce32eceb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -335,26 +335,28 @@ def test_interpreter_constraints_to_pex_info(): # constraint without interpreter class pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints=">=2.7,<3"', + '--interpreter-constraint=>=2.7', + '--interpreter-constraint=<3', '-o', pex_out_path]) if sys.version_info[0] == 3: assert res.return_code == 102 else: res.assert_success() pex_info = get_pex_info(pex_out_path) - assert '>=2.7,<3' == pex_info.interpreter_constraints + assert ['>=2.7', '<3'] == pex_info.interpreter_constraints # constraint with interpreter class pex_out_path = os.path.join(output_dir, 'pex2.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints="CPython>=2.7,<3"', + '--interpreter-constraint=CPython>=2.7', + '--interpreter-constraint=CPython<3', '-o', pex_out_path]) if sys.version_info[0] == 3 or hasattr(sys, 'pypy_version_info'): assert res.return_code == 102 else: res.assert_success() pex_info = get_pex_info(pex_out_path) - assert 'CPython>=2.7,<3' == pex_info.interpreter_constraints + assert ['CPython>=2.7', 'CPython<3'] == pex_info.interpreter_constraints @pytest.mark.skipif(NOT_CPYTHON_36) @@ -363,44 +365,48 @@ def test_interpreter_constraints_to_pex_info_py36(): # constraint without interpreter class pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints=">=3"', + '--interpreter-constraint=>=3', '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) - assert '>=3' == pex_info.interpreter_constraints + assert ['>=3'] == pex_info.interpreter_constraints # constraint with interpreter class pex_out_path = os.path.join(output_dir, 'pex2.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints="CPython>=3"', + '--interpreter-constraint=CPython>=3', '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) - assert 'CPython>=3' == pex_info.interpreter_constraints + assert ['CPython>=3'] == pex_info.interpreter_constraints -def test_resolve_interpreter_with_constraints_option(): +def test_interpreter_resolution_with_constraint_option(): with temporary_dir() as output_dir: pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints=">=2.7,<3"', + '--interpreter-constraint=>=2.7', + '--interpreter-constraint=<3', '-o', pex_out_path]) if sys.version_info[0] == 3: assert res.return_code == 102 else: res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert pex_info.build_properties['version'][0] == 2 pex_out_path = os.path.join(output_dir, 'pex2.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints=">3"', + '--interpreter-constraint=>3', '-o', pex_out_path]) if sys.version_info[0] == 3: res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert pex_info.build_properties['version'][0] == 3 else: assert res.return_code == 102 -@pytest.mark.skipif("hasattr(sys, 'pypy_version_info')") def test_interpreter_resolution_with_pex_python_path(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') @@ -408,17 +414,19 @@ def test_interpreter_resolution_with_pex_python_path(): pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path - pex_python_path = ':'.join([os.getcwd() + '/.pyenv/versions/2.7.10/bin/python', - os.getcwd() + '/.pyenv/versions/3.6.3/bin/python']) + pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) - # constraint to build pex cleanly; PPP + pex_bootstrapper.py + # constraints to build pex cleanly; PPP + pex_bootstrapper.py # will use these constraints to override sys.executable on pex re-exec - interpreter_constraint = '>3' if sys.version_info[0] == 3 else '>=2.7,<3' + interpreter_constraint1 = '>3' if sys.version_info[0] == 3 else '<3' + interpreter_constraint2 = '<3.8' if sys.version_info[0] == 3 else '>=2.7' pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraints="%s"' % interpreter_constraint, + '--interpreter-constraint=%s' % interpreter_constraint1, + '--interpreter-constraint=%s' % interpreter_constraint2, '-o', pex_out_path]) res.assert_success() @@ -429,3 +437,151 @@ def test_interpreter_resolution_with_pex_python_path(): assert str(pex_python_path.split(':')[1]).encode() in stdout else: assert str(pex_python_path.split(':')[0]).encode() in stdout + + +@pytest.mark.skipif(NOT_CPYTHON_36) +def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set both PPP and PP + pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) + pex_python = '/path/to/some/python' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python_path.split(':')[1].encode() + assert correct_interpreter_path in stdout + + +@pytest.mark.skipif(NOT_CPYTHON_36) +def test_pex_python(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + # test PEX_PYTHON + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout + + # test PEX_PYTHON with incompatible constraints + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set PEX_PYTHON + pex_python = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 1 + fail_str = 'not compatible with specified interpreter constraints'.encode() + assert fail_str in stdout + + # test PEX_PYTHON with no constraints + pex_out_path = os.path.join(td, 'pex3.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout + + +def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): + with temporary_dir() as td: + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(sys.executable).encode() in stdout + + +def test_pex_exec_with_pex_python_path_only(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set pex python path + pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + # test that pex bootstrapper selects lowest version interpreter + # in pex python path (python2.7) + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(pex_python_path.split(':')[0]).encode() in stdout + + +def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set both PPP and PP + pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) + pex_python = '/path/to/some/python' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + # test that pex bootstrapper selects lowest version interpreter + # in pex python path (python2.7) + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(pex_python_path.split(':')[0]).encode() in stdout diff --git a/tests/test_interpreter_constraints.py b/tests/test_interpreter_constraints.py deleted file mode 100644 index 2a3aa632b..000000000 --- a/tests/test_interpreter_constraints.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import pytest - -from pex.interpreter_constraints import parse_interpreter_constraints - - -def test_parse_interpreter_constraints(): - example_input = '>=2.7, <3' - assert parse_interpreter_constraints(example_input) == ['>=2.7', '<3'] - - example_input = '>=2.7,<3' - assert parse_interpreter_constraints(example_input) == ['>=2.7', '<3'] - - example_input = 'CPython>=2.7,<3' - assert parse_interpreter_constraints(example_input) == ['CPython>=2.7', 'CPython<3'] - - example_input = 'CPython>=2.7, <3' - assert parse_interpreter_constraints(example_input) == ['CPython>=2.7', 'CPython<3'] - - example_input = 'CPython==3.6' - assert parse_interpreter_constraints(example_input) == ['CPython==3.6'] - - with pytest.raises(SystemExit) as e: - example_input = 'CPython>=?.2.7,=<3>' - parse_interpreter_constraints(example_input) - assert 'Unknown requirement string' in str(e.info) - - example_input = '==2,2.7><,3_9' - parse_interpreter_constraints(example_input) - assert 'Unknown requirement string' in str(e.info) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 83179517e..c77803360 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -3,55 +3,16 @@ import os -import pytest from twitter.common.contextutil import temporary_dir from pex.common import open_zip -from pex.pex_bootstrapper import _find_compatible_interpreter_in_pex_python_path, get_pex_info -from pex.testing import write_simple_pex - -try: - import mock -except ImportError: - import unittest.mock as mock - - -@pytest.fixture -def py27_interpreter(): - mock_interpreter = mock.MagicMock() - mock_interpreter.binary = '/path/to/python2.7' - mock_interpreter.version = (2, 7, 10) - mock_interpreter.__lt__ = lambda x, y: x.version < y.version - return mock_interpreter - - -@pytest.fixture -def py36_interpreter(): - mock_interpreter = mock.MagicMock() - mock_interpreter.binary = '/path/to/python3.6' - mock_interpreter.version = (3, 6, 3) - mock_interpreter.__lt__ = lambda x, y: x.version < y.version - return mock_interpreter - - -def mock_get_python_interpreter(binary): - """Patch function for resolving PythonInterpreter mock objects from Pex Python Path""" - if '3' in binary: - return py36_interpreter() - elif '2' in binary: - return py27_interpreter() - - -def mock_matches(interpreter, filters, meet_all_constraints): - """Patch function for determining if the supplied interpreter complies with the filters""" - if '>3' in filters: - return True if interpreter.version > (3, 0, 0) else False - elif '<3' in filters: - return True if interpreter.version < (3, 0, 0) else False - elif '>=2.7' in filters: - return True if interpreter.version > (2, 7, 0) else False - else: - return False +from pex.interpreter import PythonInterpreter +from pex.pex_bootstrapper import ( + _find_compatible_interpreters, + _get_python_interpreter, + get_pex_info +) +from pex.testing import ensure_python_interpreter, write_simple_pex def test_get_pex_info(): @@ -74,19 +35,40 @@ def test_get_pex_info(): assert pex_info.dump() == pex_info_2.dump() -@mock.patch('pex.interpreter_constraints._matches', side_effect=mock_matches) -@mock.patch('pex.pex_bootstrapper._get_python_interpreter', side_effect=mock_get_python_interpreter) -def test_find_compatible_interpreter_in_python_path(mock_get_python_interpreter, mock_matches): - pex_python_path = ':'.join(['/path/to/python2.7', '/path/to/python3.6']) +def test_find_compatible_interpreters(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + + interpreters = _find_compatible_interpreters(pex_python_path, ['>3']) + assert interpreters[0].binary == pex_python_path.split(':')[1] + + interpreters = _find_compatible_interpreters(pex_python_path, ['<3']) + assert interpreters[0].binary == pex_python_path.split(':')[0] + + interpreters = _find_compatible_interpreters(pex_python_path, ['<2']) + assert not interpreters + + interpreters = _find_compatible_interpreters(pex_python_path, ['>4']) + assert not interpreters - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>3') - assert interpreter.binary == '/path/to/python3.6' + # test fallback to PATH + interpreters = _find_compatible_interpreters('', ['<3']) + assert len(interpreters) > 0 + assert all([i.version < (3, 0, 0) for i in interpreters]) + assert 'pyenv_test' not in ' '.join([i.binary for i in interpreters]) - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<3') - assert interpreter.binary == '/path/to/python2.7' + interpreters = _find_compatible_interpreters('', ['>3']) + assert len(interpreters) > 0 + assert all([i.version > (3, 0, 0) for i in interpreters]) + assert 'pyenv_test' not in ' '.join([i.binary for i in interpreters]) - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '<2') - assert interpreter is None - interpreter = _find_compatible_interpreter_in_pex_python_path(pex_python_path, '>4') - assert interpreter is None +def test_get_python_interpreter(): + ensure_python_interpreter('2.7.10') + good_binary = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python' + res1 = _get_python_interpreter(good_binary).binary + res2 = PythonInterpreter.from_binary(good_binary).binary + assert res1 == res2 + assert _get_python_interpreter('bad/path/to/binary/Python') is None From c0ef0074a97386cee068fd3374b7d1948dd94149 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 15 Nov 2017 18:35:15 -0800 Subject: [PATCH 17/43] Add validation for conflicting constraints --- pex/bin/pex.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index d4a26ae20..243333790 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,7 +23,7 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter -from pex.interpreter_constraints import check_requirements_are_well_formed, matched_interpreters +from pex.interpreter_constraints import matched_interpreters, validate_constraints from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX @@ -530,9 +530,15 @@ def build_pex(args, options, resolver_option_builder): ] if options.interpreter_constraint: + # Overwrite the current interpreter as defined by sys.executable. + # NB: options.python and interpreter constraints cannot be used together, so this will not + # affect usages of the interpreter(s) specified by the "--python" command line flag + if ENV.PEX_PYTHON_PATH: + interpreters = PythonInterpreter.all(ENV.PEX_PYTHON_PATH.split(os.pathsep)) + else: + interpreters = PythonInterpreter.all() constraints = options.interpreter_constraint - # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted - check_requirements_are_well_formed(constraints) + validate_constraints(constraints) interpreters = list(matched_interpreters(interpreters, constraints, meet_all_constraints=True)) if not interpreters: @@ -545,7 +551,6 @@ def build_pex(args, options, resolver_option_builder): # options.preamble_file is None preamble = None - # TODO: make whether to take min or max version interpreter configurable interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) @@ -619,6 +624,9 @@ def main(args=None): args, cmdline = args, [] options, reqs = parser.parse_args(args=args) + if options.python and options.interpreter_constraint: + die('The "--python" and "--interpreter constraint" options cannot be used together.') + if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: From 0ffe8c23bd32370fb714368f2e5a80003826be9a Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 15 Nov 2017 18:35:42 -0800 Subject: [PATCH 18/43] Refactor and clean up code to enfore DRY --- pex/interpreter_constraints.py | 26 ++++++------ pex/pex_bootstrapper.py | 73 ++++++++++++++++------------------ pex/pex_info.py | 4 +- pex/testing.py | 11 ++--- pex/variables.py | 7 ++-- 5 files changed, 58 insertions(+), 63 deletions(-) diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 33814cb27..08e247aa1 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -3,42 +3,40 @@ # A library of functions for filtering Python interpreters based on compatibility constraints +from .common import die from .interpreter import PythonIdentity +from .tracer import TRACER -def _matches(interpreter, filters, meet_all_constraints=False): - if meet_all_constraints: - return all(interpreter.identity.matches(filt) for filt in filters) - else: - return any(interpreter.identity.matches(filt) for filt in filters) - - -def _matching(interpreters, filters, meet_all_constraints=False): +def _matching(interpreters, constraints, meet_all_constraints=False): for interpreter in interpreters: - if _matches(interpreter, filters, meet_all_constraints): + check = all if meet_all_constraints else any + if check(interpreter.identity.matches(filt) for filt in constraints): yield interpreter -def check_requirements_are_well_formed(constraints): +def validate_constraints(constraints): + # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted # Check that the compatibility requirements are well-formed. for req in constraints: try: PythonIdentity.parse_requirement(req) except ValueError as e: - from .common import die die("Compatibility requirements are not formatted properly: %s" % str(e)) -def matched_interpreters(interpreters, filters, meet_all_constraints=False): +def matched_interpreters(interpreters, constraints, meet_all_constraints=False): """Given some filters, yield any interpreter that matches at least one of them, or all of them if meet_all_constraints is set to True. :param interpreters: a list of PythonInterpreter objects for filtering - :param filters: A sequence of strings that constrain the interpreter compatibility for this + :param constraints: A sequence of strings that constrain the interpreter compatibility for this pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class. :param meet_all_constraints: whether to match against all filters. Defaults to matching interpreters that match at least one filter. """ - for match in _matching(interpreters, filters, meet_all_constraints): + for match in _matching(interpreters, constraints, meet_all_constraints): + TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s" + % (constraints, match.binary), V=3) yield match diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index accaeaac6..5b2a92c92 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -61,14 +61,6 @@ def find_in_path(target_interpreter): return try_path -def _get_python_interpreter(binary): - try: - return PythonInterpreter.from_binary(binary) - # from_binary attempts to execute the interpreter - except Executor.ExecutionError: - return None - - def _find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. @@ -76,84 +68,87 @@ def _find_compatible_interpreters(pex_python_path, compatibility_constraints): if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): - pi = _get_python_interpreter(binary) - if pi: - interpreters.append(pi) + try: + interpreters.append(PythonInterpreter.from_binary(binary)) + except Executor.ExecutionError: + pass if not interpreters: - die('No interpreters from PEX_PYTHON_PATH can be found on the system. Exiting.') + die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: # All qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() - compatible_interpreters = list(matched_interpreters( + return list(matched_interpreters( interpreters, compatibility_constraints, meet_all_constraints=True)) - return compatible_interpreters if compatible_interpreters else None -def _handle_pex_python(target_python, compatibility_constraints): +def _select_pex_python_interpreter(target_python, compatibility_constraints): """Re-exec using the PEX_PYTHON interpreter""" target = find_in_path(target_python) + if not target: die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) elif compatibility_constraints: pi = PythonInterpreter.from_binary(target) - if not all(pi.identity.matches(constraint) for constraint in compatibility_constraints): + if not list(matched_interpreters([pi], compatibility_constraints, meet_all_constraints=True)): die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' 'interpreter constraints: %s' % (target, str(compatibility_constraints))) if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) - ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True - os.execve(target, [target_python] + sys.argv, ENV.copy()) + return target -def _handle_general_interpreter_selection(pex_python_path, compatibility_constraints): +def _select_interpreter(pex_python_path, compatibility_constraints): """Handle selection in the case that PEX_PYTHON_PATH is set or interpreter compatibility constraints are specified. """ compatible_interpreters = _find_compatible_interpreters( pex_python_path, compatibility_constraints) - target = None - if compatible_interpreters: - target = min(compatible_interpreters).binary - if not target: + if not compatible_interpreters: die('Failed to find compatible interpreter for constraints: %s' % str(compatibility_constraints)) + # TODO: https://github.com/pantsbuild/pex/issues/430 + target = min(compatible_interpreters).binary + if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): if pex_python_path: TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target) else: TRACER.log('Re-exec to interpreter %s that matches constraints: %s' % (target, str(compatibility_constraints))) - ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True - os.execve(target, [target] + sys.argv, ENV.copy()) + return target + +def maybe_reexec_pex(compatibility_constraints): -def maybe_reexec_pex(compatibility_constraints=None): if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC: return - if ENV.PEX_PYTHON and ENV.PEX_PYTHON_PATH: - # both vars are defined, fall through to PEX_PYTHON_PATH resolution (give precedence to PPP) - pass - elif ENV.PEX_PYTHON: - # preserve PEX_PYTHON re-exec for backwards compatibility - _handle_pex_python(ENV.PEX_PYTHON, compatibility_constraints) - if not compatibility_constraints: + selected_interpreter = None + if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH: + # preserve PEX_PYTHON re-exec for backwards compatibility + selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, compatibility_constraints) + elif ENV.PEX_PYTHON_PATH: + #if not compatibility_constraints: # if no compatibility constraints are specified, we want to match against # the lowest-versioned interpreter in PEX_PYTHON_PATH if it is set - if not ENV.PEX_PYTHON_PATH: + #if not ENV.PEX_PYTHON_PATH: # no PEX_PYTHON_PATH, PEX_PYTHON, or interpreter constraints, continue as normal - return - - _handle_general_interpreter_selection(ENV.PEX_PYTHON_PATH, compatibility_constraints) + #return + selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints) + if selected_interpreter: + ENV.delete('PEX_PYTHON') + ENV.delete('PEX_PYTHON_PATH') + ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True + os.execve(selected_interpreter, [selected_interpreter] + sys.argv[1:], ENV.copy()) def bootstrap_pex(entry_point): from .finders import register_finders register_finders() - pex_info = get_pex_info(entry_point) - with TRACER.timed('Bootstrapping pex with maybe_reexec_pex', V=2): + with TRACER.timed('Bootstrapping pex with maybe_reexec_pex', V=3): + pex_info = get_pex_info(entry_point) maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex diff --git a/pex/pex_info.py b/pex/pex_info.py index f1ce90524..bc81262a0 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -208,7 +208,8 @@ def interpreter_constraints(self): return self._interpreter_constraints def add_interpreter_constraint(self, value): - self._interpreter_constraints.append(str(value)) + if str(value) not in self._interpreter_constraints: + self._interpreter_constraints.append(str(value)) @property def ignore_errors(self): @@ -290,6 +291,7 @@ def update(self, other): self._pex_info.update(other._pex_info) self._distributions.update(other.distributions) self._interpreter_constraints.extend(other.interpreter_constraints) + self._interpreter_constraints = list(set(self._interpreter_constraints)) self._requirements.update(other.requirements) def dump(self, **kwargs): diff --git a/pex/testing.py b/pex/testing.py index 65098b461..79edf2fd2 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -281,8 +281,9 @@ def combine_pex_coverage(coverage_file_iter): def bootstrap_python_installer(): - install_location = os.getcwd() + '/.pyenv_test' - if not os.path.exists(install_location) or not os.path.exists(install_location + '/bin/pyenv'): + install_location = os.path.join(os.getcwd(), '.pyenv_test') + if not os.path.exists(install_location) or not os.path.exists( + os.path.join(os.getcwd(), '.pyenv_test')): for _ in range(3): try: subprocess.call(['git', 'clone', 'https://github.com/pyenv/pyenv.git', install_location]) @@ -296,7 +297,7 @@ def bootstrap_python_installer(): def ensure_python_interpreter(version): bootstrap_python_installer() - install_location = os.getcwd() + '/.pyenv_test/versions/' + version + install_location = os.path.join(os.getcwd(), '.pyenv_test/versions', version) if not os.path.exists(install_location): - os.environ['PYENV_ROOT'] = os.getcwd() + '/.pyenv_test' - subprocess.call([os.getcwd() + '/.pyenv_test/bin/pyenv', 'install', version]) + os.environ['PYENV_ROOT'] = os.path.join(os.getcwd(), '.pyenv_test') + subprocess.call([os.path.join(os.getcwd(), '.pyenv_test/bin/pyenv'), 'install', version]) diff --git a/pex/variables.py b/pex/variables.py index 44fb4cda5..571aacd58 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -235,7 +235,7 @@ def PEX_PYTHON(self): def PEX_PYTHON_PATH(self): """String - A colon-seperated string containing paths to binaires of blessed Python interpreters + A colon-seperated string containing binaires of blessed Python interpreters for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the interpreter. @@ -326,12 +326,11 @@ def SHOULD_EXIT_BOOTSTRAP_REEXEC(self): of a second execution of maybe_reexec_pex if neither PEX_PYTHON or PEX_PYTHON_PATH are set but interpreter constraints are specified. """ - value = self._environ.get('SHOULD_EXIT_REEXEC', 'False') - return False if value == 'False' else True + return bool(self._environ.get('SHOULD_EXIT_BOOTSTRAP_REEXEC', '')) @SHOULD_EXIT_BOOTSTRAP_REEXEC.setter def SHOULD_EXIT_BOOTSTRAP_REEXEC(self, value): - self._environ['SHOULD_EXIT_REEXEC'] = str(value) + self._environ['SHOULD_EXIT_BOOTSTRAP_REEXEC'] = str(value) # Global singleton environment From b5a86ea2ce3ff047ce0c3c0785143328d1b94a6c Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 13:42:28 -0800 Subject: [PATCH 19/43] Enforce interpreter selection at build time in addition to runtime selection --- pex/bin/pex.py | 10 +++--- pex/pex_bootstrapper.py | 13 ++++--- tests/test_integration.py | 66 +++++++++++----------------------- tests/test_pex_bootstrapper.py | 35 +++--------------- 4 files changed, 38 insertions(+), 86 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 243333790..5c31f6da5 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,10 +23,11 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter -from pex.interpreter_constraints import matched_interpreters, validate_constraints +from pex.interpreter_constraints import validate_constraints from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX +from pex.pex_bootstrapper import find_compatible_interpreters from pex.pex_builder import PEXBuilder from pex.platforms import Platform from pex.requirements import requirements_from_file @@ -533,13 +534,9 @@ def build_pex(args, options, resolver_option_builder): # Overwrite the current interpreter as defined by sys.executable. # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag - if ENV.PEX_PYTHON_PATH: - interpreters = PythonInterpreter.all(ENV.PEX_PYTHON_PATH.split(os.pathsep)) - else: - interpreters = PythonInterpreter.all() constraints = options.interpreter_constraint validate_constraints(constraints) - interpreters = list(matched_interpreters(interpreters, constraints, meet_all_constraints=True)) + interpreters = find_compatible_interpreters(ENV.PEX_PYTHON_PATH, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) @@ -552,6 +549,7 @@ def build_pex(args, options, resolver_option_builder): preamble = None interpreter = min(interpreters) + pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 5b2a92c92..33bd5c227 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -61,13 +61,14 @@ def find_in_path(target_interpreter): return try_path -def _find_compatible_interpreters(pex_python_path, compatibility_constraints): +def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): + try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: @@ -75,8 +76,12 @@ def _find_compatible_interpreters(pex_python_path, compatibility_constraints): if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: - # All qualifying interpreters found in $PATH - interpreters = PythonInterpreter.all() + if not os.getenv('PATH', ''): + # Use sys.executable if no PATH variable present + return [PythonInterpreter.get()] + else: + # All qualifying interpreters found in $PATH + interpreters = PythonInterpreter.all() return list(matched_interpreters( interpreters, compatibility_constraints, meet_all_constraints=True)) @@ -102,7 +107,7 @@ def _select_interpreter(pex_python_path, compatibility_constraints): """Handle selection in the case that PEX_PYTHON_PATH is set or interpreter compatibility constraints are specified. """ - compatible_interpreters = _find_compatible_interpreters( + compatible_interpreters = find_compatible_interpreters( pex_python_path, compatibility_constraints) if not compatible_interpreters: diff --git a/tests/test_integration.py b/tests/test_integration.py index bce32eceb..3e7a448a3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -332,36 +332,23 @@ def test_pex_path_in_pex_info_and_env(): def test_interpreter_constraints_to_pex_info(): with temporary_dir() as output_dir: + os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) # constraint without interpreter class pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', '-o', pex_out_path]) - if sys.version_info[0] == 3: - assert res.return_code == 102 - else: - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert ['>=2.7', '<3'] == pex_info.interpreter_constraints - - # constraint with interpreter class - pex_out_path = os.path.join(output_dir, 'pex2.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=CPython>=2.7', - '--interpreter-constraint=CPython<3', - '-o', pex_out_path]) - if sys.version_info[0] == 3 or hasattr(sys, 'pypy_version_info'): - assert res.return_code == 102 - else: - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert ['CPython>=2.7', 'CPython<3'] == pex_info.interpreter_constraints + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert ['>=2.7', '<3'] == pex_info.interpreter_constraints -@pytest.mark.skipif(NOT_CPYTHON_36) def test_interpreter_constraints_to_pex_info_py36(): with temporary_dir() as output_dir: + os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) # constraint without interpreter class pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', @@ -371,46 +358,26 @@ def test_interpreter_constraints_to_pex_info_py36(): pex_info = get_pex_info(pex_out_path) assert ['>=3'] == pex_info.interpreter_constraints - # constraint with interpreter class - pex_out_path = os.path.join(output_dir, 'pex2.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=CPython>=3', - '-o', pex_out_path]) - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert ['CPython>=3'] == pex_info.interpreter_constraints - def test_interpreter_resolution_with_constraint_option(): with temporary_dir() as output_dir: + os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', '-o', pex_out_path]) - if sys.version_info[0] == 3: - assert res.return_code == 102 - else: - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert pex_info.build_properties['version'][0] == 2 - - pex_out_path = os.path.join(output_dir, 'pex2.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=>3', - '-o', pex_out_path]) - if sys.version_info[0] == 3: - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert pex_info.build_properties['version'][0] == 3 - else: - assert res.return_code == 102 + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert ['>=2.7', '<3'] == pex_info.interpreter_constraints def test_interpreter_resolution_with_pex_python_path(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path @@ -432,6 +399,7 @@ def test_interpreter_resolution_with_pex_python_path(): stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 if sys.version_info[0] == 3: assert str(pex_python_path.split(':')[1]).encode() in stdout @@ -444,6 +412,7 @@ def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP @@ -472,7 +441,9 @@ def test_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' pexrc.write("PEX_PYTHON=%s" % pex_python) @@ -526,6 +497,7 @@ def test_pex_python(): def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', '-o', pex_out_path]) @@ -541,6 +513,7 @@ def test_pex_exec_with_pex_python_path_only(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path @@ -565,6 +538,7 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index c77803360..2a6b802fd 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -6,12 +6,7 @@ from twitter.common.contextutil import temporary_dir from pex.common import open_zip -from pex.interpreter import PythonInterpreter -from pex.pex_bootstrapper import ( - _find_compatible_interpreters, - _get_python_interpreter, - get_pex_info -) +from pex.pex_bootstrapper import find_compatible_interpreters, get_pex_info from pex.testing import ensure_python_interpreter, write_simple_pex @@ -41,34 +36,14 @@ def test_find_compatible_interpreters(): pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) - interpreters = _find_compatible_interpreters(pex_python_path, ['>3']) + interpreters = find_compatible_interpreters(pex_python_path, ['>3']) assert interpreters[0].binary == pex_python_path.split(':')[1] - interpreters = _find_compatible_interpreters(pex_python_path, ['<3']) + interpreters = find_compatible_interpreters(pex_python_path, ['<3']) assert interpreters[0].binary == pex_python_path.split(':')[0] - interpreters = _find_compatible_interpreters(pex_python_path, ['<2']) + interpreters = find_compatible_interpreters(pex_python_path, ['<2']) assert not interpreters - interpreters = _find_compatible_interpreters(pex_python_path, ['>4']) + interpreters = find_compatible_interpreters(pex_python_path, ['>4']) assert not interpreters - - # test fallback to PATH - interpreters = _find_compatible_interpreters('', ['<3']) - assert len(interpreters) > 0 - assert all([i.version < (3, 0, 0) for i in interpreters]) - assert 'pyenv_test' not in ' '.join([i.binary for i in interpreters]) - - interpreters = _find_compatible_interpreters('', ['>3']) - assert len(interpreters) > 0 - assert all([i.version > (3, 0, 0) for i in interpreters]) - assert 'pyenv_test' not in ' '.join([i.binary for i in interpreters]) - - -def test_get_python_interpreter(): - ensure_python_interpreter('2.7.10') - good_binary = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python' - res1 = _get_python_interpreter(good_binary).binary - res2 = PythonInterpreter.from_binary(good_binary).binary - assert res1 == res2 - assert _get_python_interpreter('bad/path/to/binary/Python') is None From bcfd6c4dda2b33a4e0790cc157556d1768980dfb Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 16:38:10 -0800 Subject: [PATCH 20/43] Add ability to read from pexrc at pex buildtime; fix test hermeticity to pass CI --- pex/bin/pex.py | 18 ++++++++++++++++-- pex/pex_bootstrapper.py | 10 ++-------- pex/pex_builder.py | 5 ++++- pex/pex_info.py | 13 +++++++++---- pex/variables.py | 37 ++++++++++++++++++++----------------- tests/test_integration.py | 35 +++++++++++++---------------------- 6 files changed, 64 insertions(+), 54 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 5c31f6da5..6ed858c81 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -530,13 +530,27 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] + # NB: this block is neccesary to load variables from the pexrc runtime configuration. + # It is only neccesary to cover the case where the pexrc file is located in the same + # directory as the output pex. The other supported pexrc locations (/etc/pexrc) do not + # require special treatment for reading at build time. + if options.pex_name: + if '/' in options.pex_name: # the output pex is not in the working directory + pexrc_dir = os.path.dirname(options.pex_name) + else: + pexrc_dir = os.getcwd() + else: + pexrc_dir = '' + pexrc = os.path.join(pexrc_dir, '.pexrc') + if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag constraints = options.interpreter_constraint validate_constraints(constraints) - interpreters = find_compatible_interpreters(ENV.PEX_PYTHON_PATH, constraints) + pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') + interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) @@ -559,7 +573,7 @@ def build_pex(args, options, resolver_option_builder): pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: - for ic in constraints: + for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 33bd5c227..5d791f730 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -77,10 +77,10 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: if not os.getenv('PATH', ''): - # Use sys.executable if no PATH variable present + # no $PATH, use sys.executable return [PythonInterpreter.get()] else: - # All qualifying interpreters found in $PATH + # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() return list(matched_interpreters( @@ -135,12 +135,6 @@ def maybe_reexec_pex(compatibility_constraints): # preserve PEX_PYTHON re-exec for backwards compatibility selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, compatibility_constraints) elif ENV.PEX_PYTHON_PATH: - #if not compatibility_constraints: - # if no compatibility constraints are specified, we want to match against - # the lowest-versioned interpreter in PEX_PYTHON_PATH if it is set - #if not ENV.PEX_PYTHON_PATH: - # no PEX_PYTHON_PATH, PEX_PYTHON, or interpreter constraints, continue as normal - #return selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints) if selected_interpreter: ENV.delete('PEX_PYTHON') diff --git a/pex/pex_builder.py b/pex/pex_builder.py index e98e74937..f00f1e9d7 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -76,7 +76,6 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea interpreter exit. """ self._chroot = chroot or Chroot(path or safe_mkdtemp()) - self._pex_info = pex_info or PexInfo.default() self._frozen = False self._interpreter = interpreter or PythonInterpreter.get() self._shebang = self._interpreter.identity.hashbang() @@ -84,6 +83,10 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea self._preamble = to_bytes(preamble or '') self._copy = copy self._distributions = set() + if interpreter: + self._pex_info = pex_info or PexInfo.default(interpreter) + else: + self._pex_info = pex_info or PexInfo.default() def _ensure_unfrozen(self, name='Operation'): if self._frozen: diff --git a/pex/pex_info.py b/pex/pex_info.py index bc81262a0..cedf9789c 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -51,11 +51,16 @@ class PexInfo(object): INTERNAL_CACHE = '.deps' @classmethod - def make_build_properties(cls): + def make_build_properties(cls, interpreter=None): from .interpreter import PythonInterpreter from pkg_resources import get_platform - pi = PythonInterpreter.get() + if interpreter: + # construct pex info with proper build properties if using an + # interpreter other than sys.executable + pi = interpreter + else: + pi = PythonInterpreter.get() return { 'class': pi.identity.interpreter, 'version': pi.identity.version, @@ -63,11 +68,11 @@ def make_build_properties(cls): } @classmethod - def default(cls): + def default(cls, interpreter=None): pex_info = { 'requirements': [], 'distributions': {}, - 'build_properties': cls.make_build_properties(), + 'build_properties': cls.make_build_properties(interpreter), } return cls(info=pex_info) diff --git a/pex/variables.py b/pex/variables.py index 571aacd58..3efb82e12 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -33,11 +33,30 @@ def iter_help(cls): variable_type, variable_text = cls.process_pydoc(getattr(value, '__doc__')) yield variable_name, variable_type, variable_text + @classmethod + def from_rc(cls, rc): + """Read pex runtime configuration variables from a pexrc file.""" + ret_vars = {} + for filename in ['/etc/pexrc', rc, os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')]: + try: + with open(os.path.expanduser(filename)) as fh: + rc_items = map(cls._get_kv, fh) + ret_vars.update(dict(filter(None, rc_items))) + except IOError: + continue + return ret_vars + + @classmethod + def _get_kv(cls, variable): + kv = variable.strip().split('=') + if len(list(filter(None, kv))) == 2: + return kv + def __init__(self, environ=None, rc='~/.pexrc', use_defaults=True): self._use_defaults = use_defaults self._environ = environ.copy() if environ else os.environ if not self.PEX_IGNORE_RCFILES: - rc_values = self._from_rc(rc).copy() + rc_values = self.from_rc(rc).copy() rc_values.update(self._environ) self._environ = rc_values @@ -50,22 +69,6 @@ def delete(self, variable): def set(self, variable, value): self._environ[variable] = str(value) - def _from_rc(self, rc): - ret_vars = {} - for filename in ['/etc/pexrc', rc, os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')]: - try: - with open(os.path.expanduser(filename)) as fh: - rc_items = map(self._get_kv, fh) - ret_vars.update(dict(filter(None, rc_items))) - except IOError: - continue - return ret_vars - - def _get_kv(self, variable): - kv = variable.strip().split('=') - if len(list(filter(None, kv))) == 2: - return kv - def _defaulted(self, default): return default if self._use_defaults else None diff --git a/tests/test_integration.py b/tests/test_integration.py index 3e7a448a3..1605834da 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -332,28 +332,25 @@ def test_pex_path_in_pex_info_and_env(): def test_interpreter_constraints_to_pex_info(): with temporary_dir() as output_dir: - os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + env = os.environ.copy() + env['PATH'] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) - # constraint without interpreter class + + # target python 2 pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>=2.7', '<3'] == pex_info.interpreter_constraints - -def test_interpreter_constraints_to_pex_info_py36(): - with temporary_dir() as output_dir: - os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) - # constraint without interpreter class - pex_out_path = os.path.join(output_dir, 'pex1.pex') + # target python 3 + pex_out_path = os.path.join(output_dir, 'pex2.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=3', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>=3'] == pex_info.interpreter_constraints @@ -361,23 +358,24 @@ def test_interpreter_constraints_to_pex_info_py36(): def test_interpreter_resolution_with_constraint_option(): with temporary_dir() as output_dir: - os.environ["PATH"] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', + env = os.environ.copy() + env['PATH'] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>=2.7', '<3'] == pex_info.interpreter_constraints + assert pex_info.build_properties['version'][0] < 3 def test_interpreter_resolution_with_pex_python_path(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path @@ -412,7 +410,6 @@ def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP @@ -441,14 +438,12 @@ def test_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') - with open(pexrc_path, 'w') as pexrc: pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' pexrc.write("PEX_PYTHON=%s" % pex_python) - # test PEX_PYTHON + # test PEX_PYTHON with valid constraints pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', @@ -465,7 +460,6 @@ def test_pex_python(): # test PEX_PYTHON with incompatible constraints pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: - # set PEX_PYTHON pex_python = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7' pexrc.write("PEX_PYTHON=%s" % pex_python) @@ -497,7 +491,6 @@ def test_pex_python(): def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', '-o', pex_out_path]) @@ -513,7 +506,6 @@ def test_pex_exec_with_pex_python_path_only(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path @@ -538,7 +530,6 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - os.environ["PATH"] = '/path/that/will/not/be/used' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP From d5566b549ab8aabfdd50aa55a99e9aa30b29a19b Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 16:51:25 -0800 Subject: [PATCH 21/43] More travis debug --- pex/pex_bootstrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 5d791f730..abc9044b7 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -68,7 +68,6 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): - try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: From 350deb9e2b406fbbc24455512a54d43eb1fb7aff Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:01:35 -0800 Subject: [PATCH 22/43] More travis debugging --- pex/pex_bootstrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index abc9044b7..c5036520a 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -79,6 +79,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): # no $PATH, use sys.executable return [PythonInterpreter.get()] else: + raise ValueError(os.getenv('PATH', '')) # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() From ce099b612dc68588434459fc8091489f00704d90 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:07:37 -0800 Subject: [PATCH 23/43] Travis debug 2 --- pex/pex_bootstrapper.py | 1 - tests/test_integration.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index c5036520a..abc9044b7 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -79,7 +79,6 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): # no $PATH, use sys.executable return [PythonInterpreter.get()] else: - raise ValueError(os.getenv('PATH', '')) # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() diff --git a/tests/test_integration.py b/tests/test_integration.py index 1605834da..c1e77f84b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -346,15 +346,6 @@ def test_interpreter_constraints_to_pex_info(): pex_info = get_pex_info(pex_out_path) assert ['>=2.7', '<3'] == pex_info.interpreter_constraints - # target python 3 - pex_out_path = os.path.join(output_dir, 'pex2.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=>=3', - '-o', pex_out_path], env=env) - res.assert_success() - pex_info = get_pex_info(pex_out_path) - assert ['>=3'] == pex_info.interpreter_constraints - def test_interpreter_resolution_with_constraint_option(): with temporary_dir() as output_dir: From 155d18d5bfbdefcf04dedfded6ea49b9c089c3b8 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:16:50 -0800 Subject: [PATCH 24/43] Travis debug 3 --- tests/test_integration.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index c1e77f84b..6053ff4db 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -429,6 +429,8 @@ def test_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: + env = os.environ.copy() + env['PATH'] = '/some/path' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' @@ -439,11 +441,11 @@ def test_pex_python(): res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout @@ -458,11 +460,11 @@ def test_pex_python(): res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) assert rc == 1 fail_str = 'not compatible with specified interpreter constraints'.encode() assert fail_str in stdout @@ -470,11 +472,11 @@ def test_pex_python(): # test PEX_PYTHON with no constraints pex_out_path = os.path.join(td, 'pex3.pex') res = run_pex_command(['--disable-cache', - '-o', pex_out_path]) + '-o', pex_out_path], env=env) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout From ff3e569b730b2b8d24a6e4a44672bf3e5adf9ea5 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:28:51 -0800 Subject: [PATCH 25/43] Travis test 4 --- pex/pex_bootstrapper.py | 4 +++- tests/test_integration.py | 15 ++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index abc9044b7..d8fcc443a 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -65,6 +65,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. """ + ''' if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): @@ -80,7 +81,8 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): return [PythonInterpreter.get()] else: # get all qualifying interpreters found in $PATH - interpreters = PythonInterpreter.all() + ''' + interpreters = PythonInterpreter.all() return list(matched_interpreters( interpreters, compatibility_constraints, meet_all_constraints=True)) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6053ff4db..1b262f9fb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -424,13 +424,10 @@ def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): assert correct_interpreter_path in stdout -@pytest.mark.skipif(NOT_CPYTHON_36) def test_pex_python(): ensure_python_interpreter('2.7.10') ensure_python_interpreter('3.6.3') with temporary_dir() as td: - env = os.environ.copy() - env['PATH'] = '/some/path' pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' @@ -441,11 +438,11 @@ def test_pex_python(): res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', - '-o', pex_out_path], env=env) + '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout @@ -460,11 +457,11 @@ def test_pex_python(): res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', - '-o', pex_out_path], env=env) + '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 1 fail_str = 'not compatible with specified interpreter constraints'.encode() assert fail_str in stdout @@ -472,11 +469,11 @@ def test_pex_python(): # test PEX_PYTHON with no constraints pex_out_path = os.path.join(td, 'pex3.pex') res = run_pex_command(['--disable-cache', - '-o', pex_out_path], env=env) + '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload, env=env) + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout From b7d851011454c2abb7d297d8020e4274bfcbfbfc Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:34:46 -0800 Subject: [PATCH 26/43] Travis debug 5 --- pex/bin/pex.py | 4 ++-- pex/pex_bootstrapper.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 6ed858c81..6002a9443 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -549,8 +549,8 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag constraints = options.interpreter_constraint validate_constraints(constraints) - pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') - interpreters = find_compatible_interpreters(pex_python_path, constraints) + #pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') + interpreters = find_compatible_interpreters('', constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index d8fcc443a..abc9044b7 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -65,7 +65,6 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. """ - ''' if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): @@ -81,8 +80,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): return [PythonInterpreter.get()] else: # get all qualifying interpreters found in $PATH - ''' - interpreters = PythonInterpreter.all() + interpreters = PythonInterpreter.all() return list(matched_interpreters( interpreters, compatibility_constraints, meet_all_constraints=True)) From 0b3764d1989c5e8f5b50cd67e65349ea4d7dcf23 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:44:18 -0800 Subject: [PATCH 27/43] Travis debug 4 --- pex/bin/pex.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 6002a9443..b1a48ce82 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -534,14 +534,6 @@ def build_pex(args, options, resolver_option_builder): # It is only neccesary to cover the case where the pexrc file is located in the same # directory as the output pex. The other supported pexrc locations (/etc/pexrc) do not # require special treatment for reading at build time. - if options.pex_name: - if '/' in options.pex_name: # the output pex is not in the working directory - pexrc_dir = os.path.dirname(options.pex_name) - else: - pexrc_dir = os.getcwd() - else: - pexrc_dir = '' - pexrc = os.path.join(pexrc_dir, '.pexrc') if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. @@ -549,8 +541,7 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag constraints = options.interpreter_constraint validate_constraints(constraints) - #pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') - interpreters = find_compatible_interpreters('', constraints) + interpreters = find_compatible_interpreters(ENV.PEX_PYTHON_PATH, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) From 5832b102059c96f8f8a5ea9c2e38163884688a75 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:49:47 -0800 Subject: [PATCH 28/43] Travis debug 9 --- pex/bin/pex.py | 11 ++++++++++- pex/pex_bootstrapper.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index b1a48ce82..6ed858c81 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -534,6 +534,14 @@ def build_pex(args, options, resolver_option_builder): # It is only neccesary to cover the case where the pexrc file is located in the same # directory as the output pex. The other supported pexrc locations (/etc/pexrc) do not # require special treatment for reading at build time. + if options.pex_name: + if '/' in options.pex_name: # the output pex is not in the working directory + pexrc_dir = os.path.dirname(options.pex_name) + else: + pexrc_dir = os.getcwd() + else: + pexrc_dir = '' + pexrc = os.path.join(pexrc_dir, '.pexrc') if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. @@ -541,7 +549,8 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag constraints = options.interpreter_constraint validate_constraints(constraints) - interpreters = find_compatible_interpreters(ENV.PEX_PYTHON_PATH, constraints) + pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') + interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index abc9044b7..3bacfee3c 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -75,7 +75,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: - if not os.getenv('PATH', ''): + if os.getenv('PATH', ''): # no $PATH, use sys.executable return [PythonInterpreter.get()] else: From 2b5012d3748d5232fe7eb5c7a10d8194b0fb5bf5 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 17:53:43 -0800 Subject: [PATCH 29/43] Travis debug 7 --- pex/pex_bootstrapper.py | 2 +- tests/test_integration.py | 55 --------------------------------------- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 3bacfee3c..abc9044b7 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -75,7 +75,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: - if os.getenv('PATH', ''): + if not os.getenv('PATH', ''): # no $PATH, use sys.executable return [PythonInterpreter.get()] else: diff --git a/tests/test_integration.py b/tests/test_integration.py index 1b262f9fb..c37cce009 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -424,61 +424,6 @@ def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): assert correct_interpreter_path in stdout -def test_pex_python(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') - with temporary_dir() as td: - pexrc_path = os.path.join(td, '.pexrc') - with open(pexrc_path, 'w') as pexrc: - pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' - pexrc.write("PEX_PYTHON=%s" % pex_python) - - # test PEX_PYTHON with valid constraints - pex_out_path = os.path.join(td, 'pex.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=>3', - '--interpreter-constraint=<3.8', - '-o', pex_out_path]) - res.assert_success() - - stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) - assert rc == 0 - correct_interpreter_path = pex_python.encode() - assert correct_interpreter_path in stdout - - # test PEX_PYTHON with incompatible constraints - pexrc_path = os.path.join(td, '.pexrc') - with open(pexrc_path, 'w') as pexrc: - pex_python = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7' - pexrc.write("PEX_PYTHON=%s" % pex_python) - - pex_out_path = os.path.join(td, 'pex2.pex') - res = run_pex_command(['--disable-cache', - '--interpreter-constraint=>3', - '--interpreter-constraint=<3.8', - '-o', pex_out_path]) - res.assert_success() - - stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) - assert rc == 1 - fail_str = 'not compatible with specified interpreter constraints'.encode() - assert fail_str in stdout - - # test PEX_PYTHON with no constraints - pex_out_path = os.path.join(td, 'pex3.pex') - res = run_pex_command(['--disable-cache', - '-o', pex_out_path]) - res.assert_success() - - stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' - stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) - assert rc == 0 - correct_interpreter_path = pex_python.encode() - assert correct_interpreter_path in stdout - - def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): with temporary_dir() as td: pex_out_path = os.path.join(td, 'pex.pex') From 575a8ad05409b970e670b20473757b41ba99b9ae Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 20 Nov 2017 18:11:11 -0800 Subject: [PATCH 30/43] Travis debug 8 --- pex/bin/pex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 6ed858c81..5fa30c8f5 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -546,7 +546,7 @@ def build_pex(args, options, resolver_option_builder): if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. # NB: options.python and interpreter constraints cannot be used together, so this will not - # affect usages of the interpreter(s) specified by the "--python" command line flag + # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') From ce0dac67499cfe8c322986efd2896c54a1c8921f Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 21 Nov 2017 09:51:04 -0800 Subject: [PATCH 31/43] Add travis PATH variable to yml --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 01ecd2aa3..3d1a4f0e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ dist: precise # TRAVIS_PYTHON_VERSION +env: + global: + - PATH=/home/travis/build/pantsbuild/pex/.tox/py36/bin:/home/travis/build/pantsbuild/pex/.tox/py27/bin + cache: directories: - .pyenv_test From a2db930aae36654fe50b2c07a9435136051edf84 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 21 Nov 2017 10:09:49 -0800 Subject: [PATCH 32/43] Add pex python testing for backwards compatibility --- tests/test_integration.py | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index c37cce009..670f0503b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -485,3 +485,58 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 assert str(pex_python_path.split(':')[0]).encode() in stdout + + +def test_pex_python(): + ensure_python_interpreter('2.7.10') + ensure_python_interpreter('3.6.3') + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + # test PEX_PYTHON with valid constraints + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout + + # test PEX_PYTHON with incompatible constraints + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + pex_python = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 1 + fail_str = 'not compatible with specified interpreter constraints'.encode() + assert fail_str in stdout + + # test PEX_PYTHON with no constraints + pex_out_path = os.path.join(td, 'pex3.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout From 1150b657ecd066b903d7319a232e78f38ac74380 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 21 Nov 2017 10:20:11 -0800 Subject: [PATCH 33/43] Readd once problematic tests now that travis has custom set PATH env var --- tests/test_integration.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 670f0503b..cfa15f8dc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -332,31 +332,33 @@ def test_pex_path_in_pex_info_and_env(): def test_interpreter_constraints_to_pex_info(): with temporary_dir() as output_dir: - env = os.environ.copy() - env['PATH'] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) - # target python 2 pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', - '-o', pex_out_path], env=env) + '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>=2.7', '<3'] == pex_info.interpreter_constraints + # target python 3 + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '-o', pex_out_path]) + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert ['>3'] == pex_info.interpreter_constraints + def test_interpreter_resolution_with_constraint_option(): with temporary_dir() as output_dir: - env = os.environ.copy() - env['PATH'] = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) pex_out_path = os.path.join(output_dir, 'pex1.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>=2.7', '--interpreter-constraint=<3', - '-o', pex_out_path], env=env) + '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>=2.7', '<3'] == pex_info.interpreter_constraints From 3a143d1ab3ae35fdd015ef9c0d9f103a8489b8bd Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 22 Nov 2017 15:45:16 -0800 Subject: [PATCH 34/43] Refactor methods and clean up doc strings/literals --- .travis.yml | 2 ++ pex/bin/pex.py | 10 +++++----- pex/interpreter_constraints.py | 10 ++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d1a4f0e0..8a90ebbd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ dist: precise env: global: + # This is necessary to have consistent testing of methods that rely on falling back to PATH for + # selecting a python interpreter. - PATH=/home/travis/build/pantsbuild/pex/.tox/py36/bin:/home/travis/build/pantsbuild/pex/.tox/py27/bin cache: diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 5fa30c8f5..0285eeaa2 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -530,8 +530,8 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] - # NB: this block is neccesary to load variables from the pexrc runtime configuration. - # It is only neccesary to cover the case where the pexrc file is located in the same + # NB: this block is necessary to load variables from the pexrc runtime configuration. + # It is only necessary to cover the case where the pexrc file is located in the same # directory as the output pex. The other supported pexrc locations (/etc/pexrc) do not # require special treatment for reading at build time. if options.pex_name: @@ -539,9 +539,9 @@ def build_pex(args, options, resolver_option_builder): pexrc_dir = os.path.dirname(options.pex_name) else: pexrc_dir = os.getcwd() + pexrc = os.path.join(pexrc_dir, '.pexrc') else: - pexrc_dir = '' - pexrc = os.path.join(pexrc_dir, '.pexrc') + pexrc = '.pexrc' if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. @@ -637,7 +637,7 @@ def main(args=None): options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: - die('The "--python" and "--interpreter constraint" options cannot be used together.') + die('The "--python" and "--interpreter-constraint" options cannot be used together.') if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 08e247aa1..7e9539aee 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -36,7 +36,9 @@ def matched_interpreters(interpreters, constraints, meet_all_constraints=False): :param meet_all_constraints: whether to match against all filters. Defaults to matching interpreters that match at least one filter. """ - for match in _matching(interpreters, constraints, meet_all_constraints): - TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s" - % (constraints, match.binary), V=3) - yield match + check = all if meet_all_constraints else any + for interpreter in interpreters: + if check(interpreter.identity.matches(filt) for filt in constraints): + TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s" + % (constraints, interpreter.binary), V=3) + yield interpreter From b045ba5ad618e11b673b32d93acd16381b92a82c Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 22 Nov 2017 15:46:01 -0800 Subject: [PATCH 35/43] Refactor testing methods for readability, refactor TRACER logic in pex bootstrapper to be cleaner and simpler --- pex/pex_bootstrapper.py | 58 +++++++++++++++++++++++++---------------- pex/testing.py | 1 + pex/variables.py | 10 ++++--- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index abc9044b7..bac77e450 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -63,7 +63,7 @@ def find_in_path(target_interpreter): def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use - PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH. + PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] @@ -77,7 +77,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): else: if not os.getenv('PATH', ''): # no $PATH, use sys.executable - return [PythonInterpreter.get()] + interpreters = [PythonInterpreter.get()] else: # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() @@ -87,7 +87,6 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): def _select_pex_python_interpreter(target_python, compatibility_constraints): - """Re-exec using the PEX_PYTHON interpreter""" target = find_in_path(target_python) if not target: @@ -97,15 +96,12 @@ def _select_pex_python_interpreter(target_python, compatibility_constraints): if not list(matched_interpreters([pi], compatibility_constraints, meet_all_constraints=True)): die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' 'interpreter constraints: %s' % (target, str(compatibility_constraints))) - if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): - TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) - return target + if not os.path.exists(target): + die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target) + return target def _select_interpreter(pex_python_path, compatibility_constraints): - """Handle selection in the case that PEX_PYTHON_PATH is set or interpreter compatibility - constraints are specified. - """ compatible_interpreters = find_compatible_interpreters( pex_python_path, compatibility_constraints) @@ -116,38 +112,54 @@ def _select_interpreter(pex_python_path, compatibility_constraints): target = min(compatible_interpreters).binary if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): - if pex_python_path: - TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target) - else: - TRACER.log('Re-exec to interpreter %s that matches constraints: %s' % (target, - str(compatibility_constraints))) return target def maybe_reexec_pex(compatibility_constraints): + """ + Handle environment overrides for the Python interpreter to use when executing this pex. + This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO + metadata. If PEX_PYTHON is set in a pexrc, it attempts to obtain the binary location of the + interpreter specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for + a matching interpreter in accordance with the interpreter constraints. If both variables are + present in a pexrc, this function gives precedence to PEX_PYTHON_PATH and errors out if no + compatible interpreters can be found on said path. If neither variable is set, fall through to + plain pex execution using the currently executing Python interpreter. + + :param compatibility_constraints: list of requirements-style strings that constrain the + Python interpreter to re-exec this pex with. + + """ if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC: return selected_interpreter = None - if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH: - # preserve PEX_PYTHON re-exec for backwards compatibility - selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, compatibility_constraints) - elif ENV.PEX_PYTHON_PATH: - selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints) + with TRACER.timed('Selecting runtime interpreter based on pexrc', V=3): + if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH: + # preserve PEX_PYTHON re-exec for backwards compatibility + selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, + compatibility_constraints) + elif ENV.PEX_PYTHON_PATH: + selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints) + if selected_interpreter: ENV.delete('PEX_PYTHON') ENV.delete('PEX_PYTHON_PATH') ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True - os.execve(selected_interpreter, [selected_interpreter] + sys.argv[1:], ENV.copy()) + cmdline = [selected_interpreter] + sys.argv[1:] + TRACER.log('Re-executing: cmdline="%s", sys.executable="%s", PEX_PYTHON="%s", ' + 'PEX_PYTHON_PATH="%s", COMPATIBILITY_CONSTRAINTS="%s"' + % (cmdline, sys.executable, ENV.PEX_PYTHON, ENV.PEX_PYTHON_PATH, + compatibility_constraints)) + os.execve(selected_interpreter, cmdline, ENV.copy()) def bootstrap_pex(entry_point): from .finders import register_finders register_finders() - with TRACER.timed('Bootstrapping pex with maybe_reexec_pex', V=3): - pex_info = get_pex_info(entry_point) - maybe_reexec_pex(pex_info.interpreter_constraints) + pex_info = get_pex_info(entry_point) + maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex pex.PEX(entry_point).execute() diff --git a/pex/testing.py b/pex/testing.py index 79edf2fd2..964a8a3ae 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -301,3 +301,4 @@ def ensure_python_interpreter(version): if not os.path.exists(install_location): os.environ['PYENV_ROOT'] = os.path.join(os.getcwd(), '.pyenv_test') subprocess.call([os.path.join(os.getcwd(), '.pyenv_test/bin/pyenv'), 'install', version]) + return os.path.join(install_location, 'bin', 'python' + version[0:3]) diff --git a/pex/variables.py b/pex/variables.py index 3efb82e12..c18a5c14c 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -35,7 +35,11 @@ def iter_help(cls): @classmethod def from_rc(cls, rc): - """Read pex runtime configuration variables from a pexrc file.""" + """Read pex runtime configuration variables from a pexrc file. + + :param rc: an absolute path to a pexrc file + :return ret_vars: a dict object containing key values pairs found in processed pexrc files + """ ret_vars = {} for filename in ['/etc/pexrc', rc, os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')]: try: @@ -238,7 +242,7 @@ def PEX_PYTHON(self): def PEX_PYTHON_PATH(self): """String - A colon-seperated string containing binaires of blessed Python interpreters + A colon-separated string containing binaires of blessed Python interpreters for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the interpreter. @@ -320,7 +324,7 @@ def SHOULD_EXIT_BOOTSTRAP_REEXEC(self): """Boolean Whether to re-exec in maybe_reexec_pex function of pex_bootstrapper.py. Default: false. - This is neccesary because that function relies on checking against variables present in the + This is necessary because that function relies on checking against variables present in the ENV to determine whether to re-exec or return to current execution. When we introduced interpreter constraints to a pex, these constraints can influence interpreter selection without the need for a PEX_PYTHON or PEX_PYTHON_PATH ENV variable set. Since re-exec previously checked From 66ef8ee6dddb0854d7641bc0b469f484a184c90f Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 22 Nov 2017 15:46:23 -0800 Subject: [PATCH 36/43] Refactor tests to improve readability --- tests/test_integration.py | 32 ++++++++++++++++---------------- tests/test_pex_bootstrapper.py | 21 +++++++++++++++------ 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index cfa15f8dc..49a052613 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -366,14 +366,14 @@ def test_interpreter_resolution_with_constraint_option(): def test_interpreter_resolution_with_pex_python_path(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path - pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) # constraints to build pex cleanly; PPP + pex_bootstrapper.py @@ -400,14 +400,14 @@ def test_interpreter_resolution_with_pex_python_path(): @pytest.mark.skipif(NOT_CPYTHON_36) def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP - pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) pex_python = '/path/to/some/python' pexrc.write("PEX_PYTHON=%s" % pex_python) @@ -440,14 +440,14 @@ def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): def test_pex_exec_with_pex_python_path_only(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set pex python path - pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) pex_out_path = os.path.join(td, 'pex.pex') @@ -464,14 +464,14 @@ def test_pex_exec_with_pex_python_path_only(): def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: # set both PPP and PP - pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) pex_python = '/path/to/some/python' pexrc.write("PEX_PYTHON=%s" % pex_python) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 2a6b802fd..557f5c2ed 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -6,6 +6,7 @@ from twitter.common.contextutil import temporary_dir from pex.common import open_zip +from pex.interpreter import PythonInterpreter from pex.pex_bootstrapper import find_compatible_interpreters, get_pex_info from pex.testing import ensure_python_interpreter, write_simple_pex @@ -31,19 +32,27 @@ def test_get_pex_info(): def test_find_compatible_interpreters(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') - pex_python_path = ':'.join([os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7', - os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6']) + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.9'), + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('2.7.11'), + ensure_python_interpreter('3.6.3') + ]) interpreters = find_compatible_interpreters(pex_python_path, ['>3']) - assert interpreters[0].binary == pex_python_path.split(':')[1] + assert interpreters[0].binary == pex_python_path.split(':')[3] # 3.6.3 interpreters = find_compatible_interpreters(pex_python_path, ['<3']) - assert interpreters[0].binary == pex_python_path.split(':')[0] + assert interpreters[0].binary == pex_python_path.split(':')[0] # 2.7.9 interpreters = find_compatible_interpreters(pex_python_path, ['<2']) assert not interpreters interpreters = find_compatible_interpreters(pex_python_path, ['>4']) assert not interpreters + + interpreters = find_compatible_interpreters(pex_python_path, ['<2.7.11, >2.7.9']) + assert interpreters[0].binary == pex_python_path.split(':')[1] # 2.7.10 + + interpreters = find_compatible_interpreters('', ['<3']) + assert interpreters[0] in PythonInterpreter.all() # All interpreters on PATH From 6a2bd58b3becba1243f5848466a6b039950f9cd5 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 22 Nov 2017 19:07:12 -0800 Subject: [PATCH 37/43] Move build time pex rc reading logic into Variables and add more granular testing for the interpreter selection method in pex bootstrapper --- pex/bin/pex.py | 16 ++-------------- pex/pex_bootstrapper.py | 2 +- pex/variables.py | 20 ++++++++++++++++---- tests/test_pex_bootstrapper.py | 14 +++++++++++++- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 0285eeaa2..482151423 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -530,26 +530,14 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] - # NB: this block is necessary to load variables from the pexrc runtime configuration. - # It is only necessary to cover the case where the pexrc file is located in the same - # directory as the output pex. The other supported pexrc locations (/etc/pexrc) do not - # require special treatment for reading at build time. - if options.pex_name: - if '/' in options.pex_name: # the output pex is not in the working directory - pexrc_dir = os.path.dirname(options.pex_name) - else: - pexrc_dir = os.getcwd() - pexrc = os.path.join(pexrc_dir, '.pexrc') - else: - pexrc = '.pexrc' - if options.interpreter_constraint: # Overwrite the current interpreter as defined by sys.executable. # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) - pex_python_path = Variables.from_rc(pexrc).get('PEX_PYTHON_PATH', '') + rc_variables = Variables.from_rc('', build_time_rc_dir=os.path.dirname(options.pex_name)) + pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index bac77e450..da92a18bb 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -71,7 +71,7 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: - pass + TRACER.log("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: diff --git a/pex/variables.py b/pex/variables.py index c18a5c14c..ccafe209a 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -34,14 +34,26 @@ def iter_help(cls): yield variable_name, variable_type, variable_text @classmethod - def from_rc(cls, rc): + def from_rc(cls, rc, build_time_rc_dir=''): """Read pex runtime configuration variables from a pexrc file. - :param rc: an absolute path to a pexrc file - :return ret_vars: a dict object containing key values pairs found in processed pexrc files + :param rc: an absolute path to a pexrc file + :param build_time_rc_dir: an absolute path to a directory containing a pexrc. This is needed to + read a pexrc at build time in the case where a pexrc is located in the same directory as a the + output pex. + + :return ret_vars: a dict object containing key values pairs found in processed pexrc files """ ret_vars = {} - for filename in ['/etc/pexrc', rc, os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')]: + rc_locations = ['/etc/pexrc', + rc, + os.path.join(os.path.dirname(sys.argv[0]), '.pexrc'), + os.getcwd()] + if not rc: + rc_locations.append('~/.pexrc') + if build_time_rc_dir: + rc_locations.append(os.path.join(build_time_rc_dir, '.pexrc')) + for filename in rc_locations: try: with open(os.path.expanduser(filename)) as fh: rc_items = map(cls._get_kv, fh) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 557f5c2ed..9592e8fe2 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -36,15 +36,27 @@ def test_find_compatible_interpreters(): ensure_python_interpreter('2.7.9'), ensure_python_interpreter('2.7.10'), ensure_python_interpreter('2.7.11'), + ensure_python_interpreter('3.4.2'), + ensure_python_interpreter('3.5.4'), + ensure_python_interpreter('3.6.2'), ensure_python_interpreter('3.6.3') ]) interpreters = find_compatible_interpreters(pex_python_path, ['>3']) - assert interpreters[0].binary == pex_python_path.split(':')[3] # 3.6.3 + assert interpreters[0].binary == pex_python_path.split(':')[3] # 3.4.2 interpreters = find_compatible_interpreters(pex_python_path, ['<3']) assert interpreters[0].binary == pex_python_path.split(':')[0] # 2.7.9 + interpreters = find_compatible_interpreters(pex_python_path, ['>3.5.4']) + assert interpreters[0].binary == pex_python_path.split(':')[5] # 3.6.2 + + interpreters = find_compatible_interpreters(pex_python_path, ['>3.4.2, <3.6']) + assert interpreters[0].binary == pex_python_path.split(':')[4] # 3.5.4 + + interpreters = find_compatible_interpreters(pex_python_path, ['>3.6.2']) + assert interpreters[0].binary == pex_python_path.split(':')[6] # 3.6.3 + interpreters = find_compatible_interpreters(pex_python_path, ['<2']) assert not interpreters From 83116198b9fdc8dad3dda3118fe6d6d5222324f3 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 28 Nov 2017 13:02:09 -0800 Subject: [PATCH 38/43] Add default arg to from_rc --- pex/bin/pex.py | 2 +- pex/variables.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 482151423..888477719 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -536,7 +536,7 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) - rc_variables = Variables.from_rc('', build_time_rc_dir=os.path.dirname(options.pex_name)) + rc_variables = Variables.from_rc(build_time_rc_dir=os.path.dirname(options.pex_name)) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) diff --git a/pex/variables.py b/pex/variables.py index ccafe209a..38e5fbc6a 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -34,7 +34,7 @@ def iter_help(cls): yield variable_name, variable_type, variable_text @classmethod - def from_rc(cls, rc, build_time_rc_dir=''): + def from_rc(cls, rc='~/.pexrc', build_time_rc_dir=None): """Read pex runtime configuration variables from a pexrc file. :param rc: an absolute path to a pexrc file From e7590c22502914157b6d9155c6a5e25aa17dbae3 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 29 Nov 2017 13:18:01 -0800 Subject: [PATCH 39/43] Remove rc param from Variables constructor and modify related tests. Cleanup docs and refactor a few one-liners. --- pex/bin/pex.py | 6 ++++-- pex/interpreter_constraints.py | 7 ------- pex/pex_bootstrapper.py | 4 ++-- pex/pex_builder.py | 5 +---- pex/pex_info.py | 7 +------ pex/variables.py | 21 +++++++++------------ tests/test_variables.py | 16 ++++++++++------ 7 files changed, 27 insertions(+), 39 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 888477719..f410e6169 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -531,12 +531,14 @@ def build_pex(args, options, resolver_option_builder): ] if options.interpreter_constraint: - # Overwrite the current interpreter as defined by sys.executable. # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) - rc_variables = Variables.from_rc(build_time_rc_dir=os.path.dirname(options.pex_name)) + # Special accommodations are needed at build time to read a pexrc that is in the same + # directory as the output pex. + pexrc = os.path.join(os.path.dirname(options.pex_name), '.pexrc') + rc_variables = Variables.from_rc(build_time_rc=pexrc) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 7e9539aee..32648971c 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -8,13 +8,6 @@ from .tracer import TRACER -def _matching(interpreters, constraints, meet_all_constraints=False): - for interpreter in interpreters: - check = all if meet_all_constraints else any - if check(interpreter.identity.matches(filt) for filt in constraints): - yield interpreter - - def validate_constraints(constraints): # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted # Check that the compatibility requirements are well-formed. diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index da92a18bb..46eff3f2d 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -91,7 +91,7 @@ def _select_pex_python_interpreter(target_python, compatibility_constraints): if not target: die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) - elif compatibility_constraints: + if compatibility_constraints: pi = PythonInterpreter.from_binary(target) if not list(matched_interpreters([pi], compatibility_constraints, meet_all_constraints=True)): die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' @@ -125,7 +125,7 @@ def maybe_reexec_pex(compatibility_constraints): a matching interpreter in accordance with the interpreter constraints. If both variables are present in a pexrc, this function gives precedence to PEX_PYTHON_PATH and errors out if no compatible interpreters can be found on said path. If neither variable is set, fall through to - plain pex execution using the currently executing Python interpreter. + plain pex execution using PATH searching or the currently executing interpreter. :param compatibility_constraints: list of requirements-style strings that constrain the Python interpreter to re-exec this pex with. diff --git a/pex/pex_builder.py b/pex/pex_builder.py index f00f1e9d7..dd9c9a028 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -83,10 +83,7 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea self._preamble = to_bytes(preamble or '') self._copy = copy self._distributions = set() - if interpreter: - self._pex_info = pex_info or PexInfo.default(interpreter) - else: - self._pex_info = pex_info or PexInfo.default() + self._pex_info = pex_info or PexInfo.default(interpreter) def _ensure_unfrozen(self, name='Operation'): if self._frozen: diff --git a/pex/pex_info.py b/pex/pex_info.py index cedf9789c..a5ba6f56b 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -55,12 +55,7 @@ def make_build_properties(cls, interpreter=None): from .interpreter import PythonInterpreter from pkg_resources import get_platform - if interpreter: - # construct pex info with proper build properties if using an - # interpreter other than sys.executable - pi = interpreter - else: - pi = PythonInterpreter.get() + pi = interpreter or PythonInterpreter.get() return { 'class': pi.identity.interpreter, 'version': pi.identity.version, diff --git a/pex/variables.py b/pex/variables.py index 38e5fbc6a..e1c3bf5ff 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -34,11 +34,10 @@ def iter_help(cls): yield variable_name, variable_type, variable_text @classmethod - def from_rc(cls, rc='~/.pexrc', build_time_rc_dir=None): + def from_rc(cls, build_time_rc=None): """Read pex runtime configuration variables from a pexrc file. - :param rc: an absolute path to a pexrc file - :param build_time_rc_dir: an absolute path to a directory containing a pexrc. This is needed to + :param build_time_rc: an absolute path to a pexrc. This is needed to read a pexrc at build time in the case where a pexrc is located in the same directory as a the output pex. @@ -46,13 +45,11 @@ def from_rc(cls, rc='~/.pexrc', build_time_rc_dir=None): """ ret_vars = {} rc_locations = ['/etc/pexrc', - rc, + '~/.pexrc', os.path.join(os.path.dirname(sys.argv[0]), '.pexrc'), - os.getcwd()] - if not rc: - rc_locations.append('~/.pexrc') - if build_time_rc_dir: - rc_locations.append(os.path.join(build_time_rc_dir, '.pexrc')) + os.path.join(os.getcwd(), '.pexrc')] + if build_time_rc: + rc_locations.append(build_time_rc) for filename in rc_locations: try: with open(os.path.expanduser(filename)) as fh: @@ -68,11 +65,11 @@ def _get_kv(cls, variable): if len(list(filter(None, kv))) == 2: return kv - def __init__(self, environ=None, rc='~/.pexrc', use_defaults=True): + def __init__(self, environ=None, use_defaults=True): self._use_defaults = use_defaults self._environ = environ.copy() if environ else os.environ if not self.PEX_IGNORE_RCFILES: - rc_values = self.from_rc(rc).copy() + rc_values = self.from_rc().copy() rc_values.update(self._environ) self._environ = rc_values @@ -254,7 +251,7 @@ def PEX_PYTHON(self): def PEX_PYTHON_PATH(self): """String - A colon-separated string containing binaires of blessed Python interpreters + A colon-separated string containing paths of blessed Python interpreters for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the interpreter. diff --git a/tests/test_variables.py b/tests/test_variables.py index 3a33fbede..8cd508598 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -1,6 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import os import pytest from pex.util import named_temporary_file @@ -84,27 +85,30 @@ def test_pex_get_kv(): def test_pex_from_rc(): - with named_temporary_file(mode='w') as pexrc: + with open('.pexrc', 'w') as pexrc: pexrc.write('HELLO=42') pexrc.flush() - v = Variables(rc=pexrc.name) + v = Variables() assert v._get_int('HELLO') == 42 + os.remove('.pexrc') def test_pexrc_precedence(): - with named_temporary_file(mode='w') as pexrc: + with open('.pexrc', 'w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'HELLO': 42}, rc=pexrc.name) + v = Variables(environ={'HELLO': 42}) assert v._get_int('HELLO') == 42 + os.remove('.pexrc') def test_rc_ignore(): - with named_temporary_file(mode='w') as pexrc: + with open('.pexrc', 'w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'PEX_IGNORE_RCFILES': 'True'}, rc=pexrc.name) + v = Variables(environ={'PEX_IGNORE_RCFILES': 'True'}) assert 'HELLO' not in v._environ + os.remove('.pexrc') def test_pex_vars_defaults_stripped(): From 6b3e5510950b99d3a0a83847c82e9cd45d65b8db Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Wed, 29 Nov 2017 16:47:42 -0800 Subject: [PATCH 40/43] Add a --rcfile flag to allow pexrc usage in integration tests --- pex/bin/pex.py | 16 ++++++++++++---- pex/pex_info.py | 13 ++++++------- pex/variables.py | 11 ++++------- tests/test_integration.py | 19 +++++++++++++------ tests/test_variables.py | 2 +- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index f410e6169..64f93733d 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -302,6 +302,14 @@ def configure_clp_pex_environment(parser): 'for requirements agnostic to interpreter class. This option can be passed multiple ' 'times.') + group.add_option( + '--rcfile', + dest='rc_file', + default=None, + help='A path to a pexrc file. NOTE: this flag is for testing purposes only. It is to ' + 'be used in the case that a pexrc lives in the same directory as the created ' + 'output pex as is the case with the pex integration tests.') + group.add_option( '--python-shebang', dest='python_shebang', @@ -535,10 +543,10 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) - # Special accommodations are needed at build time to read a pexrc that is in the same - # directory as the output pex. - pexrc = os.path.join(os.path.dirname(options.pex_name), '.pexrc') - rc_variables = Variables.from_rc(build_time_rc=pexrc) + pexrc = None + if options.rc_file: + pexrc = options.rc_file + rc_variables = Variables.from_rc(rc=pexrc) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) diff --git a/pex/pex_info.py b/pex/pex_info.py index a5ba6f56b..eae310a9a 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -123,7 +123,8 @@ def __init__(self, info=None): '%s of type %s' % (info, type(info))) self._pex_info = info or {} self._distributions = self._pex_info.get('distributions', {}) - self._interpreter_constraints = self._pex_info.get('interpreter_constraints', []) + # cast as set because pex info from json must store interpreter_constraints as a list + self._interpreter_constraints = set(self._pex_info.get('interpreter_constraints', set())) requirements = self._pex_info.get('requirements', []) if not isinstance(requirements, (list, tuple)): raise ValueError('Expected requirements to be a list, got %s' % type(requirements)) @@ -205,11 +206,10 @@ def interpreter_constraints(self): This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH for a list of compatible interpreters. """ - return self._interpreter_constraints + return list(self._interpreter_constraints) def add_interpreter_constraint(self, value): - if str(value) not in self._interpreter_constraints: - self._interpreter_constraints.append(str(value)) + self._interpreter_constraints.add(str(value)) @property def ignore_errors(self): @@ -290,14 +290,13 @@ def update(self, other): raise TypeError('Cannot merge a %r with PexInfo' % type(other)) self._pex_info.update(other._pex_info) self._distributions.update(other.distributions) - self._interpreter_constraints.extend(other.interpreter_constraints) - self._interpreter_constraints = list(set(self._interpreter_constraints)) + self._interpreter_constraints.update(other.interpreter_constraints) self._requirements.update(other.requirements) def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) - pex_info_copy['interpreter_constraints'] = self._interpreter_constraints + pex_info_copy['interpreter_constraints'] = list(self._interpreter_constraints) pex_info_copy['distributions'] = self._distributions.copy() return json.dumps(pex_info_copy, **kwargs) diff --git a/pex/variables.py b/pex/variables.py index e1c3bf5ff..cb665d7b6 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -34,13 +34,10 @@ def iter_help(cls): yield variable_name, variable_type, variable_text @classmethod - def from_rc(cls, build_time_rc=None): + def from_rc(cls, rc=None): """Read pex runtime configuration variables from a pexrc file. - :param build_time_rc: an absolute path to a pexrc. This is needed to - read a pexrc at build time in the case where a pexrc is located in the same directory as a the - output pex. - + :param rc: an absolute path to a pexrc file. :return ret_vars: a dict object containing key values pairs found in processed pexrc files """ ret_vars = {} @@ -48,8 +45,8 @@ def from_rc(cls, build_time_rc=None): '~/.pexrc', os.path.join(os.path.dirname(sys.argv[0]), '.pexrc'), os.path.join(os.getcwd(), '.pexrc')] - if build_time_rc: - rc_locations.append(build_time_rc) + if rc: + rc_locations.append(rc) for filename in rc_locations: try: with open(os.path.expanduser(filename)) as fh: diff --git a/tests/test_integration.py b/tests/test_integration.py index 49a052613..d504cf0f5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -340,7 +340,7 @@ def test_interpreter_constraints_to_pex_info(): '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) - assert ['>=2.7', '<3'] == pex_info.interpreter_constraints + assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints) # target python 3 pex_out_path = os.path.join(output_dir, 'pex1.pex') @@ -361,7 +361,7 @@ def test_interpreter_resolution_with_constraint_option(): '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) - assert ['>=2.7', '<3'] == pex_info.interpreter_constraints + assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints) assert pex_info.build_properties['version'][0] < 3 @@ -383,6 +383,7 @@ def test_interpreter_resolution_with_pex_python_path(): pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, '--interpreter-constraint=%s' % interpreter_constraint1, '--interpreter-constraint=%s' % interpreter_constraint2, '-o', pex_out_path]) @@ -414,6 +415,7 @@ def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', '-o', pex_out_path]) @@ -452,6 +454,7 @@ def test_pex_exec_with_pex_python_path_only(): pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, '-o', pex_out_path]) res.assert_success() @@ -478,6 +481,7 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, '-o', pex_out_path]) res.assert_success() @@ -501,6 +505,7 @@ def test_pex_python(): # test PEX_PYTHON with valid constraints pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', '-o', pex_out_path]) @@ -520,9 +525,10 @@ def test_pex_python(): pex_out_path = os.path.join(td, 'pex2.pex') res = run_pex_command(['--disable-cache', - '--interpreter-constraint=>3', - '--interpreter-constraint=<3.8', - '-o', pex_out_path]) + '--rcfile=%s' % pexrc_path, + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' @@ -534,7 +540,8 @@ def test_pex_python(): # test PEX_PYTHON with no constraints pex_out_path = os.path.join(td, 'pex3.pex') res = run_pex_command(['--disable-cache', - '-o', pex_out_path]) + '--rcfile=%s' % pexrc_path, + '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' diff --git a/tests/test_variables.py b/tests/test_variables.py index 8cd508598..ccfd6ee09 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -2,9 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os + import pytest -from pex.util import named_temporary_file from pex.variables import Variables From 34773e51975db4df95e6dd28e1db3606302b18a0 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Thu, 30 Nov 2017 09:54:13 -0800 Subject: [PATCH 41/43] Add comment regarding PEX_PYTHON --- pex/pex_bootstrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 46eff3f2d..547b373c8 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -138,6 +138,8 @@ def maybe_reexec_pex(compatibility_constraints): with TRACER.timed('Selecting runtime interpreter based on pexrc', V=3): if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH: # preserve PEX_PYTHON re-exec for backwards compatibility + # TODO: Kill this off completely in favor of PEX_PYTHON_PATH + # https://github.com/pantsbuild/pex/issues/431 selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, compatibility_constraints) elif ENV.PEX_PYTHON_PATH: From 508226f496a1dff085442b6723aecbf228422c42 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Fri, 1 Dec 2017 10:56:46 -0800 Subject: [PATCH 42/43] Add comments to interpreter constraints and cleanup docstring --- pex/interpreter_constraints.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 32648971c..dfd3dd2ef 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -1,4 +1,4 @@ -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). # A library of functions for filtering Python interpreters based on compatibility constraints @@ -9,9 +9,10 @@ def validate_constraints(constraints): - # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted - # Check that the compatibility requirements are well-formed. + # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted: + # https://github.com/pantsbuild/pex/issues/432 for req in constraints: + # Check that the compatibility requirements are well-formed. try: PythonIdentity.parse_requirement(req) except ValueError as e: @@ -28,6 +29,7 @@ def matched_interpreters(interpreters, constraints, meet_all_constraints=False): for requirements agnostic to interpreter class. :param meet_all_constraints: whether to match against all filters. Defaults to matching interpreters that match at least one filter. + :return interpreter: returns a generator that yields compatible interpreters """ check = all if meet_all_constraints else any for interpreter in interpreters: From 524456b21aeaf23ade348cbe18105a8209ad8117 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Fri, 1 Dec 2017 16:28:45 -0800 Subject: [PATCH 43/43] Final cleanups to docs and minor refactoring for tests --- pex/bin/pex.py | 10 +++------- pex/pex_bootstrapper.py | 5 +++-- pex/variables.py | 10 +++++----- tests/test_integration.py | 6 ++---- tests/test_variables.py | 18 +++++++----------- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 64f93733d..af00c1c6e 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -306,9 +306,8 @@ def configure_clp_pex_environment(parser): '--rcfile', dest='rc_file', default=None, - help='A path to a pexrc file. NOTE: this flag is for testing purposes only. It is to ' - 'be used in the case that a pexrc lives in the same directory as the created ' - 'output pex as is the case with the pex integration tests.') + help='An additional path to a pexrc file to read during configuration parsing. ' + 'Used primarily for testing.') group.add_option( '--python-shebang', @@ -543,10 +542,7 @@ def build_pex(args, options, resolver_option_builder): # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) - pexrc = None - if options.rc_file: - pexrc = options.rc_file - rc_variables = Variables.from_rc(rc=pexrc) + rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 547b373c8..3c6aa3c45 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -1,6 +1,6 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - +from __future__ import print_function import os import sys @@ -71,7 +71,8 @@ def find_compatible_interpreters(pex_python_path, compatibility_constraints): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: - TRACER.log("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary) + print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, + file=sys.stderr) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: diff --git a/pex/variables.py b/pex/variables.py index cb665d7b6..d1d3d5867 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -38,13 +38,13 @@ def from_rc(cls, rc=None): """Read pex runtime configuration variables from a pexrc file. :param rc: an absolute path to a pexrc file. - :return ret_vars: a dict object containing key values pairs found in processed pexrc files + :return: A dict of key value pairs found in processed pexrc files. + :rtype: dict """ ret_vars = {} rc_locations = ['/etc/pexrc', '~/.pexrc', - os.path.join(os.path.dirname(sys.argv[0]), '.pexrc'), - os.path.join(os.getcwd(), '.pexrc')] + os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')] if rc: rc_locations.append(rc) for filename in rc_locations: @@ -62,11 +62,11 @@ def _get_kv(cls, variable): if len(list(filter(None, kv))) == 2: return kv - def __init__(self, environ=None, use_defaults=True): + def __init__(self, environ=None, rc=None, use_defaults=True): self._use_defaults = use_defaults self._environ = environ.copy() if environ else os.environ if not self.PEX_IGNORE_RCFILES: - rc_values = self.from_rc().copy() + rc_values = self.from_rc(rc).copy() rc_values.update(self._environ) self._environ = rc_values diff --git a/tests/test_integration.py b/tests/test_integration.py index d504cf0f5..c5ea290fd 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -494,12 +494,10 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): def test_pex_python(): - ensure_python_interpreter('2.7.10') - ensure_python_interpreter('3.6.3') with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: - pex_python = os.getcwd() + '/.pyenv_test/versions/3.6.3/bin/python3.6' + pex_python = ensure_python_interpreter('3.6.3') pexrc.write("PEX_PYTHON=%s" % pex_python) # test PEX_PYTHON with valid constraints @@ -520,7 +518,7 @@ def test_pex_python(): # test PEX_PYTHON with incompatible constraints pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: - pex_python = os.getcwd() + '/.pyenv_test/versions/2.7.10/bin/python2.7' + pex_python = ensure_python_interpreter('2.7.10') pexrc.write("PEX_PYTHON=%s" % pex_python) pex_out_path = os.path.join(td, 'pex2.pex') diff --git a/tests/test_variables.py b/tests/test_variables.py index ccfd6ee09..9cc48fa27 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -1,10 +1,9 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import os - import pytest +from pex.util import named_temporary_file from pex.variables import Variables @@ -85,30 +84,27 @@ def test_pex_get_kv(): def test_pex_from_rc(): - with open('.pexrc', 'w') as pexrc: + with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=42') pexrc.flush() - v = Variables() + v = Variables(rc=pexrc.name) assert v._get_int('HELLO') == 42 - os.remove('.pexrc') def test_pexrc_precedence(): - with open('.pexrc', 'w') as pexrc: + with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'HELLO': 42}) + v = Variables(rc=pexrc.name, environ={'HELLO': 42}) assert v._get_int('HELLO') == 42 - os.remove('.pexrc') def test_rc_ignore(): - with open('.pexrc', 'w') as pexrc: + with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'PEX_IGNORE_RCFILES': 'True'}) + v = Variables(rc=pexrc.name, environ={'PEX_IGNORE_RCFILES': 'True'}) assert 'HELLO' not in v._environ - os.remove('.pexrc') def test_pex_vars_defaults_stripped():