From 7654a4c762247a94760b1d593e84b1ca710a3c34 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 4 Aug 2015 16:00:28 -0600 Subject: [PATCH 01/20] Fix from_env to capture only explicitly set values. Previously from_env also captured Variables defaults which had the effect of defaulted Variables always over-riding user supplied PexInfo values. Testing Done: CI went green here: https://travis-ci.org/pantsbuild/pex/builds/72339390 Bugs closed: 135, 136 Reviewed at https://rbcommons.com/s/twitter/r/2517/ --- CHANGES.rst | 8 ++++++++ pex/pex_info.py | 22 ++++++++++++---------- pex/variables.py | 19 +++++++++++++++---- tests/test_pex_info.py | 31 +++++++++++++++++++++++++++++++ tests/test_variables.py | 17 +++++++++++++++++ 5 files changed, 83 insertions(+), 14 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b6f904677..6d6e0dda5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ CHANGES ======= +---------- +1.0.2.dev0 +---------- + +* Bug fix: PEX-INFO values were overridden by environment `Variables` with default values that were + not explicitly set in the environment. + Fixes `#135 `_. + ----- 1.0.1 ----- diff --git a/pex/pex_info.py b/pex/pex_info.py index 17ba97792..d4e88242e 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -88,16 +88,18 @@ def from_json(cls, content): @classmethod def from_env(cls, env=ENV): + supplied_env = env.strip_defaults() + zip_safe = None if supplied_env.PEX_FORCE_LOCAL is None else not supplied_env.PEX_FORCE_LOCAL pex_info = { - 'pex_root': env.PEX_ROOT, - 'entry_point': env.PEX_MODULE, - 'script': env.PEX_SCRIPT, - 'zip_safe': not env.PEX_FORCE_LOCAL, - 'inherit_path': env.PEX_INHERIT_PATH, - 'ignore_errors': env.PEX_IGNORE_ERRORS, - 'always_write_cache': env.PEX_ALWAYS_CACHE, + 'pex_root': supplied_env.PEX_ROOT, + 'entry_point': supplied_env.PEX_MODULE, + 'script': supplied_env.PEX_SCRIPT, + 'zip_safe': zip_safe, + 'inherit_path': supplied_env.PEX_INHERIT_PATH, + 'ignore_errors': supplied_env.PEX_IGNORE_ERRORS, + 'always_write_cache': supplied_env.PEX_ALWAYS_CACHE, } - # Filter out empty entries. + # Filter out empty entries not explicitly set in the environment. return cls(info=dict((k, v) for (k, v) in pex_info.items() if v is not None)) @classmethod @@ -260,10 +262,10 @@ def update(self, other): self._distributions.update(other.distributions) self._requirements.update(other.requirements) - def dump(self): + def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) - return json.dumps(pex_info_copy) + return json.dumps(pex_info_copy, **kwargs) def copy(self): return PexInfo(info=self._pex_info.copy()) diff --git a/pex/variables.py b/pex/variables.py index 2923a62e9..059459751 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -32,8 +32,9 @@ def iter_help(cls): variable_type, variable_text = cls.process_pydoc(getattr(value, '__doc__')) yield variable_name, variable_type, variable_text - def __init__(self, environ=None): + def __init__(self, environ=None, use_defaults=True): self._environ = environ.copy() if environ is not None else os.environ + self._use_defaults = use_defaults def copy(self): return self._environ.copy() @@ -44,6 +45,9 @@ def delete(self, variable): def set(self, variable, value): self._environ[variable] = str(value) + def _defaulted(self, default): + return default if self._use_defaults else None + def _get_bool(self, variable, default=False): value = self._environ.get(variable) if value is not None: @@ -54,10 +58,10 @@ def _get_bool(self, variable, default=False): else: die('Invalid value for %s, must be 0/1/false/true, got %r' % (variable, value)) else: - return default + return self._defaulted(default) def _get_string(self, variable, default=None): - return self._environ.get(variable, default) + return self._environ.get(variable, self._defaulted(default)) def _get_path(self, variable, default=None): value = self._get_string(variable, default=default) @@ -70,7 +74,14 @@ def _get_int(self, variable, default=None): except ValueError: die('Invalid value for %s, must be an integer, got %r' % (variable, self._environ[variable])) except KeyError: - return default + return self._defaulted(default) + + def strip_defaults(self): + """Returns a copy of these variables but with defaults stripped. + + Any variables not explicitly set in the environment will have a value of `None`. + """ + return Variables(environ=self.copy(), use_defaults=False) @contextmanager def patch(self, **kw): diff --git a/tests/test_pex_info.py b/tests/test_pex_info.py index dd7f2e664..032b221da 100644 --- a/tests/test_pex_info.py +++ b/tests/test_pex_info.py @@ -5,6 +5,7 @@ from pex.orderedset import OrderedSet from pex.pex_info import PexInfo +from pex.variables import Variables def make_pex_info(requirements): @@ -32,3 +33,33 @@ def test_backwards_incompatible_pex_info(): ['world==0.2', False, None], ]) assert pi.requirements == OrderedSet(['hello==0.1', 'world==0.2']) + + +def assert_same_info(expected, actual): + assert expected.dump(sort_keys=True) == actual.dump(sort_keys=True) + + +def test_from_empty_env(): + environ = Variables(environ={}) + info = {} + assert_same_info(PexInfo(info=info), PexInfo.from_env(env=environ)) + + +def test_from_env(): + environ = dict(PEX_ROOT='/pex_root', + PEX_MODULE='entry:point', + PEX_SCRIPT='script.sh', + PEX_FORCE_LOCAL='true', + PEX_INHERIT_PATH='true', + PEX_IGNORE_ERRORS='true', + PEX_ALWAYS_CACHE='true') + + info = dict(pex_root='/pex_root', + entry_point='entry:point', + script='script.sh', + zip_safe=False, + inherit_path=True, + ignore_errors=True, + always_write_cache=True) + + assert_same_info(PexInfo(info=info), PexInfo.from_env(env=Variables(environ=environ))) diff --git a/tests/test_variables.py b/tests/test_variables.py index 2852678ca..e97698567 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -72,3 +72,20 @@ def test_pex_vars_set(): assert v._get_int('HELLO') == 42 v.delete('HELLO') assert v._get_int('HELLO') is None + + +def test_pex_vars_defaults_stripped(): + v = Variables(environ={}) + stripped = v.strip_defaults() + + # bool + assert v.PEX_ALWAYS_CACHE is not None + assert stripped.PEX_ALWAYS_CACHE is None + + # string + assert v.PEX_PATH is not None + assert stripped.PEX_PATH is None + + # int + assert v.PEX_VERBOSE is not None + assert stripped.PEX_VERBOSE is None From f6c1c26c292c947f22ee482e39003e0c5ad16183 Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 4 Aug 2015 15:25:37 -0700 Subject: [PATCH 02/20] Address #141. Update version.py to reflect 1.0.2.dev0. --- CHANGES.rst | 8 ++++++++ pex/pex.py | 6 +----- pex/version.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6d6e0dda5..4bd766da0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,14 @@ CHANGES not explicitly set in the environment. Fixes `#135 `_. +* Bug fix: Since `69649c1 `_ we have been unpatching + the side-effects of ``sys.modules`` after ``PEX.execute``. This takes all modules imported during + the PEX lifecycle and sets all their attributes to ``None``. Unfortunately, ``sys.excepthook``, + ``atexit`` and ``__del__`` may still try to operate using these tainted modules, causing exceptions + on interpreter teardown. This reverts just the ``sys`` unpatching so that the abovementioned + teardown hooks behave more predictably. + Fixes `#141 `_. + ----- 1.0.1 ----- diff --git a/pex/pex.py b/pex/pex.py index b9e86021b..2744bb61d 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -243,11 +243,7 @@ def patch_all(path, path_importer_cache, modules): new_sys_path, new_sys_path_importer_cache, new_sys_modules = cls.minimum_sys() patch_all(new_sys_path, new_sys_path_importer_cache, new_sys_modules) - - try: - yield - finally: - patch_all(old_sys_path, old_sys_path_importer_cache, old_sys_modules) + yield def _wrap_coverage(self, runner, *args): if not self._vars.PEX_COVERAGE and self._vars.PEX_COVERAGE_FILENAME is None: diff --git a/pex/version.py b/pex/version.py index 342aff87f..46818e3b4 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.1' +__version__ = '1.0.2.dev0' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' From 0a6578863585b50863b1a5dc1fea00a727ff5d12 Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 4 Aug 2015 15:56:41 -0700 Subject: [PATCH 03/20] Release 1.0.2 --- pex/version.py | 2 +- scripts/coverage.sh | 2 +- tox.ini | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pex/version.py b/pex/version.py index 46818e3b4..a44de6450 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.2.dev0' +__version__ = '1.0.2' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 680676cb0..095b7a525 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -4,4 +4,4 @@ coverage run -p -m py.test tests coverage run -p -m pex.bin.pex -v --help >&/dev/null coverage run -p -m pex.bin.pex -v -- scripts/do_nothing.py coverage run -p -m pex.bin.pex -v requests -- scripts/do_nothing.py -coverage run -p -m pex.bin.pex -v . setuptools -- scripts/do_nothing.py +coverage run -p -m pex.bin.pex -v . 'setuptools>=2.2,<16' -- scripts/do_nothing.py diff --git a/tox.ini b/tox.ini index 1c5064e9c..e370f8191 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ deps = requests: responses cachecontrol: CacheControl cachecontrol: lockfile - coverage: coverage + coverage: coverage==3.7.1 [integration] commands = @@ -55,7 +55,7 @@ commands = [testenv:coverage] basepython = python2.7 deps = - coverage + coverage==3.7.1 tox commands = # meta From cc1b3a1a94596bc855757e4ea08b3488d0d20248 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 9 Aug 2015 21:22:12 -0600 Subject: [PATCH 04/20] Accomodate OSX `Python` python binaries. --- pex/interpreter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pex/interpreter.py b/pex/interpreter.py index 0ae3b5dc0..c3ea95c53 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -178,7 +178,10 @@ def __hash__(self): class PythonInterpreter(object): REGEXEN = ( re.compile(r'jython$'), - re.compile(r'python$'), + + # NB: OSX ships python binaries named Python so we allow for capital-P. + re.compile(r'[Pp]ython$'), + re.compile(r'python[23].[0-9]$'), re.compile(r'pypy$'), re.compile(r'pypy-1.[0-9]$'), From 82c36daea414870d5275e8087a1c398752e522fe Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 10 Aug 2015 08:48:01 -0600 Subject: [PATCH 05/20] Bump the pre-release version and update the change log. --- CHANGES.rst | 11 ++++++++++- pex/version.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4bd766da0..241701309 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,9 +3,18 @@ CHANGES ======= ---------- -1.0.2.dev0 +1.0.3.dev0 ---------- +* Bug fix: Accommodate OSX `Python` python binaries. Previously the OSX python distributions shipped + with OSX, XCode and available via https://www.python.org/downloads/ could fail to be detected using + the `PythonInterpreter` class. + Fixes `#144 `_. + +----- +1.0.2 +----- + * Bug fix: PEX-INFO values were overridden by environment `Variables` with default values that were not explicitly set in the environment. Fixes `#135 `_. diff --git a/pex/version.py b/pex/version.py index a44de6450..4b53cf13b 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.2' +__version__ = '1.0.3.dev0' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' From 78fbb6b4138fbbb95f90a2c7d268c06c3daf8ef9 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Thu, 6 Aug 2015 16:08:27 -0700 Subject: [PATCH 06/20] Remove unnecessary stderr print on SystemExit().code == None --- pex/pex.py | 2 +- tests/test_pex.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pex/pex.py b/pex/pex.py index 2744bb61d..826dc13e7 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -321,7 +321,7 @@ def execute(self): except SystemExit as se: # Print a SystemExit error message, avoiding a traceback in python3. # This must happen here, as sys.stderr is about to be torn down - if not isinstance(se.code, int): + if not isinstance(se.code, int) and se.code is not None: print(se.code, file=sys.stderr) raise finally: diff --git a/tests/test_pex.py b/tests/test_pex.py index 14bfa5454..39152135d 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -60,6 +60,14 @@ def test_pex_sys_exit_prints_non_numeric_value_no_traceback(): _test_sys_exit(sys_exit_arg, expected_output, 1) +def test_pex_sys_exit_doesnt_print_none(): + _test_sys_exit('', to_bytes(''), 0) + + +def test_pex_sys_exit_prints_objects(): + _test_sys_exit('Exception("derp")', to_bytes('derp\n'), 1) + + @pytest.mark.skipif('hasattr(sys, "pypy_version_info")') def test_pex_atexit_swallowing(): body = textwrap.dedent(""" From 3aa144560362a001b59c0a4cf190fe0001cd1993 Mon Sep 17 00:00:00 2001 From: Andy Freeland Date: Tue, 11 Aug 2015 02:32:26 -0400 Subject: [PATCH 07/20] Fix a logging typo when determining the minimum sys.path --- pex/pex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/pex.py b/pex/pex.py index 2744bb61d..682371cef 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -160,7 +160,7 @@ def all_distribution_paths(path): TRACER.log('Tainted path element: %s' % path_element) site_distributions.update(all_distribution_paths(path_element)) else: - TRACER.log('Not a tained path element: %s' % path_element, V=2) + TRACER.log('Not a tainted path element: %s' % path_element, V=2) user_site_distributions.update(all_distribution_paths(USER_SITE)) From bc0edf4f09fbff847344559b37cd8ab8f290a96d Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 11 Aug 2015 12:45:31 -0700 Subject: [PATCH 08/20] Fix #139: PEX_SCRIPT fails for scripts from not-zip-safe eggs. h/t @palexander for doing the heavy lifting to figure out what was going on here. --- CHANGES.rst | 6 ++++++ pex/finders.py | 14 ++++++++++++-- pex/pex.py | 2 +- tests/test_pex.py | 8 +++++--- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 241701309..0d3fd7920 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,12 @@ CHANGES the `PythonInterpreter` class. Fixes `#144 `_. +* Bug fix: PEX_SCRIPT failed when the script was from a not-zip-safe egg. + Original PR `#139 `_. + +* Bug fix: `sys.exit` called without arguments would cause `None` to be printed on stderr since pex 1.0.1. + `#143 `_. + ----- 1.0.2 ----- diff --git a/pex/finders.py b/pex/finders.py index 4521e023f..eb3be61f5 100644 --- a/pex/finders.py +++ b/pex/finders.py @@ -262,9 +262,19 @@ def get_script_from_whl(name, dist): def get_script_from_distribution(name, dist): - if isinstance(dist._provider, FixedEggMetadata): + # PathMetadata: exploded distribution on disk. + if isinstance(dist._provider, pkg_resources.PathMetadata): + if dist.egg_info.endswith('EGG-INFO'): + return get_script_from_egg(name, dist) + elif dist.egg_info.endswith('.dist-info'): + return get_script_from_whl(name, dist) + else: + return None, None + # FixedEggMetadata: Zipped egg + elif isinstance(dist._provider, FixedEggMetadata): return get_script_from_egg(name, dist) - elif isinstance(dist._provider, (WheelMetadata, pkg_resources.PathMetadata)): + # WheelMetadata: Zipped whl (in theory should not experience this at runtime.) + elif isinstance(dist._provider, WheelMetadata): return get_script_from_whl(name, dist) return None, None diff --git a/pex/pex.py b/pex/pex.py index 6c0a37010..0e6195e5e 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -392,7 +392,7 @@ def execute_content(cls, name, content, argv0=None): try: ast = compile(content, name, 'exec', flags=0, dont_inherit=1) except SyntaxError: - die('Unable to parse %s. PEX script support only supports Python scripts.') + die('Unable to parse %s. PEX script support only supports Python scripts.' % name) old_name, old_file = globals().get('__name__'), globals().get('__file__') try: old_argv0, sys.argv[0] = sys.argv[0], argv0 diff --git a/tests/test_pex.py b/tests/test_pex.py index 39152135d..a54abce77 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -120,17 +120,19 @@ def test_minimum_sys_modules(): assert tainted_module.__path__ == ['good_path'] +@pytest.mark.parametrize('zip_safe', (False, True)) @pytest.mark.parametrize('project_name', ('my_project', 'my-project')) @pytest.mark.parametrize('installer_impl', (EggInstaller, WheelInstaller)) -def test_pex_script(installer_impl, project_name): - with make_installer(name=project_name, installer_impl=installer_impl) as installer: +def test_pex_script(installer_impl, project_name, zip_safe): + kw = dict(name=project_name, installer_impl=installer_impl, zip_safe=zip_safe) + with make_installer(**kw) as installer: bdist = DistributionHelper.distribution_from_path(installer.bdist()) env_copy = os.environ.copy() env_copy['PEX_SCRIPT'] = 'hello_world' so, rc = run_simple_pex_test('', env=env_copy) assert rc == 1, so.decode('utf-8') - assert b'Could not find' in so + assert b'Could not find script hello_world' in so so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist]) assert rc == 0, so.decode('utf-8') From 178f995d962b7e5d975f9786f0d7808d235560ec Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 11 Aug 2015 12:48:57 -0700 Subject: [PATCH 09/20] Release 1.0.3 --- CHANGES.rst | 6 +++--- pex/version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0d3fd7920..a7f31a25a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,9 +2,9 @@ CHANGES ======= ----------- -1.0.3.dev0 ----------- +----- +1.0.3 +----- * Bug fix: Accommodate OSX `Python` python binaries. Previously the OSX python distributions shipped with OSX, XCode and available via https://www.python.org/downloads/ could fail to be detected using diff --git a/pex/version.py b/pex/version.py index 4b53cf13b..c7d1aeac0 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.3.dev0' +__version__ = '1.0.3' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' From 6c8b150a2b6c0a6c01ffc89f4fdbda5b03241a3c Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 11 Aug 2015 16:09:26 -0700 Subject: [PATCH 10/20] Normalize all names in ResolvableSet. Fixes #147. --- CHANGES.rst | 8 ++++++++ pex/resolver.py | 16 ++++++++++++---- pex/version.py | 2 +- tests/test_resolver.py | 10 +++++----- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a7f31a25a..bc2eebb1e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,14 @@ CHANGES ======= +---------- +1.1.0.dev0 +---------- + +* Bug fix: We did not normalize package names in ``ResolvableSet``, so it was possible to depend on + ``sphinx`` and ``Sphinx-1.4a0.tar.gz`` and get two versions build and included into the pex. + `#147 `_. + ----- 1.0.3 ----- diff --git a/pex/resolver.py b/pex/resolver.py index c8db1cf5c..fe45fa6a2 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -8,6 +8,8 @@ import time from collections import namedtuple +from pkg_resources import safe_name + from .common import safe_mkdir from .fetcher import Fetcher from .interpreter import PythonInterpreter @@ -56,6 +58,10 @@ def merge(self, other): class _ResolvableSet(object): + @classmethod + def normalize(cls, name): + return safe_name(name).lower() + def __init__(self, tuples=None): # A list of _ResolvedPackages self.__tuples = tuples or [] @@ -71,7 +77,7 @@ def _collapse(self): # adversely but could be the source of subtle resolution quirks. resolvables = {} for resolved_packages in self.__tuples: - key = resolved_packages.resolvable.name + key = self.normalize(resolved_packages.resolvable.name) previous = resolvables.get(key, _ResolvedPackages.empty()) if previous.resolvable is None: resolvables[key] = resolved_packages @@ -86,7 +92,7 @@ def render_resolvable(resolved_packages): '(from: %s)' % resolved_packages.parent if resolved_packages.parent else '') return ', '.join( render_resolvable(resolved_packages) for resolved_packages in self.__tuples - if resolved_packages.resolvable.name == name) + if self.normalize(resolved_packages.resolvable.name) == self.normalize(name)) def _check(self): # Check whether or not the resolvables in this set are satisfiable, raise an exception if not. @@ -102,7 +108,8 @@ def merge(self, resolvable, packages, parent=None): def get(self, name): """Get the set of compatible packages given a resolvable name.""" - resolvable, packages, parent = self._collapse().get(name, _ResolvedPackages.empty()) + resolvable, packages, parent = self._collapse().get( + self.normalize(name), _ResolvedPackages.empty()) return packages def packages(self): @@ -111,7 +118,8 @@ def packages(self): def extras(self, name): return set.union( - *[set(tup.resolvable.extras()) for tup in self.__tuples if tup.resolvable.name == name]) + *[set(tup.resolvable.extras()) for tup in self.__tuples + if self.normalize(tup.resolvable.name) == self.normalize(name)]) def replace_built(self, built_packages): """Return a copy of this resolvable set but with built packages. diff --git a/pex/version.py b/pex/version.py index c7d1aeac0..ef3d5144f 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.3' +__version__ = '1.1.0.dev0' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 5b8897081..ce94d02fb 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -53,18 +53,21 @@ def test_resolvable_set(): rs = _ResolvableSet() rq = ResolvableRequirement.from_string('foo[ext]', builder) source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz') - binary_pkg = EggPackage.from_href('foo-2.3.4-py3.4.egg') + binary_pkg = EggPackage.from_href('Foo-2.3.4-py3.4.egg') rs.merge(rq, [source_pkg, binary_pkg]) - assert rs.get('foo') == set([source_pkg, binary_pkg]) + assert rs.get(source_pkg.name) == set([source_pkg, binary_pkg]) + assert rs.get(binary_pkg.name) == set([source_pkg, binary_pkg]) assert rs.packages() == [(rq, set([source_pkg, binary_pkg]), None)] # test methods assert rs.extras('foo') == set(['ext']) + assert rs.extras('Foo') == set(['ext']) # test filtering rs.merge(rq, [source_pkg]) assert rs.get('foo') == set([source_pkg]) + assert rs.get('Foo') == set([source_pkg]) with pytest.raises(Unsatisfiable): rs.merge(rq, [binary_pkg]) @@ -88,6 +91,3 @@ def test_resolvable_set_built(): updated_rs.merge(rq, [binary_pkg]) assert updated_rs.get('foo') == set([binary_pkg]) assert updated_rs.packages() == [(rq, set([binary_pkg]), None)] - - -# TODO(wickman) Write more than simple resolver test. From f18f55cb35b146c21ef783a76c26bba0a4fcb60c Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Wed, 12 Aug 2015 16:01:52 -0700 Subject: [PATCH 11/20] Fix the docs release headers. --- docs/conf.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a02eafc28..cdf0a4939 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,14 +12,17 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys +from datetime import datetime # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) +from pex.version import __version__ as PEX_VERSION + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -47,16 +50,17 @@ # General information about the project. project = u'pex' -copyright = u'2014, Pants project contributors' +copyright = u'%s, Pants project contributors' % datetime.now().year # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.7' +version = '.'.join(PEX_VERSION.split('.')[0:2]) + # The full version, including alpha/beta/rc tags. -release = '0.7.0-rc0' +release = PEX_VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From ecc767d8d0c3315057c6cd487eac82811be4c307 Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 11 Aug 2015 13:58:57 -0700 Subject: [PATCH 12/20] Add pex-identifying User-Agent to requests sessions. --- CHANGES.rst | 2 ++ pex/http.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index bc2eebb1e..12b445d47 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,8 @@ CHANGES ``sphinx`` and ``Sphinx-1.4a0.tar.gz`` and get two versions build and included into the pex. `#147 `_. +* Adds a pex-identifying User-Agent. `#101 `_. + ----- 1.0.3 ----- diff --git a/pex/http.py b/pex/http.py index 8c6474a81..116661553 100644 --- a/pex/http.py +++ b/pex/http.py @@ -13,6 +13,7 @@ from .compatibility import PY3, AbstractClass from .tracer import TRACER from .variables import ENV +from .version import __version__ as PEX_VERSION try: import requests @@ -185,6 +186,7 @@ def close(self): class RequestsContext(Context): """A requests-based Context.""" + USER_AGENT = 'pex/%s' % PEX_VERSION @staticmethod def _create_session(max_retries): @@ -192,7 +194,6 @@ def _create_session(max_retries): retrying_adapter = requests.adapters.HTTPAdapter(max_retries=max_retries) session.mount('http://', retrying_adapter) session.mount('https://', retrying_adapter) - return session def __init__(self, session=None, verify=True, env=ENV): @@ -214,7 +215,9 @@ def open(self, link): return open(link.path, 'rb') # noqa: T802 for attempt in range(self._max_retries + 1): try: - return StreamFilelike(self._session.get(link.url, verify=self._verify, stream=True), link) + return StreamFilelike(self._session.get( + link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT}), + link) except requests.exceptions.ReadTimeout: # Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not # so we'll retry them ourselves. From 884e599703045db86d022fd467e92ae26380df17 Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Wed, 12 Aug 2015 16:16:48 -0700 Subject: [PATCH 13/20] Fix missed mock of safe_mkdir. Pin pytest to 2.5.2. --- tests/test_util.py | 43 +++++++++++++++++++++++-------------------- tox.ini | 2 +- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index fd76a2ec8..99867e95f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -87,36 +87,39 @@ def test_zipsafe(): assert DistributionHelper.zipsafe(dist) is zip_safe -def test_access_zipped_assets(): - try: - import __builtin__ - builtin_path = __builtin__.__name__ - except ImportError: - # Python3 - import builtins - builtin_path = builtins.__name__ +try: + import __builtin__ as python_builtins +except ImportError: + import builtins as python_builtins + + +@mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True) +@mock.patch('pex.util.safe_mkdir', autospec=True, spec_set=True) +@mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True) +@mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True) +@mock.patch('pex.util.resource_string', autospec=True, spec_set=True) +def test_access_zipped_assets( + mock_resource_string, + mock_resource_isdir, + mock_resource_listdir, + mock_safe_mkdir, + mock_safe_mkdtemp): mock_open = mock.mock_open() - with nested(mock.patch('%s.open' % builtin_path, mock_open, create=True), - mock.patch('pex.util.resource_string', autospec=True, spec_set=True), - mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True), - mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True), - mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True)) as ( - _, mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdtemp): - - mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir']) - mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']]) - mock_resource_isdir.side_effect = iter([False, True, False]) - mock_resource_string.return_value = 'testing' + mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir']) + mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']]) + mock_resource_isdir.side_effect = iter([False, True, False]) + mock_resource_string.return_value = 'testing' + with mock.patch('%s.open' % python_builtins.__name__, mock_open, create=True): temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil') - assert mock_resource_listdir.call_count == 2 assert mock_open.call_count == 2 file_handle = mock_open.return_value.__enter__.return_value assert file_handle.write.call_count == 2 assert mock_safe_mkdtemp.mock_calls == [mock.call()] assert temp_dir == 'tmpJIMMEH' + assert mock_safe_mkdir.mock_calls == [mock.call(os.path.join('tmpJIMMEH', 'directory'))] def test_access_zipped_assets_integration(): diff --git a/tox.ini b/tox.ini index e370f8191..16240df5d 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ envlist = commands = py.test {posargs:} deps = - pytest + pytest==2.5.2 twitter.common.contextutil>=0.3.1,<0.4.0 twitter.common.dirutil>=0.3.1,<0.4.0 twitter.common.lang>=0.3.1,<0.4.0 From a253efd4a4d9f33f5ac7f6f397902f62fb1e517f Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Thu, 23 Apr 2015 12:02:23 -0700 Subject: [PATCH 14/20] Initial implementation of bdist_pex. --- pex/commands/__init__.py | 0 pex/commands/bdist_pex.py | 70 +++++++++++++++++++++++++++++++++++++++ pex/installer.py | 2 +- pex/pex_builder.py | 13 ++++++-- pex/pex_info.py | 3 +- setup.py | 4 +++ 6 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 pex/commands/__init__.py create mode 100644 pex/commands/bdist_pex.py diff --git a/pex/commands/__init__.py b/pex/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py new file mode 100644 index 000000000..f61c96512 --- /dev/null +++ b/pex/commands/bdist_pex.py @@ -0,0 +1,70 @@ +import os +import pprint +from distutils import log + +from pex.bin.pex import configure_clp, build_pex +from pex.common import die + +from setuptools import Command + + +class bdist_pex(Command): + description = "create a PEX file from a source distribution" + + user_options = [ + ('bdist-all', None, 'pexify all defined entry points'), + ('bdist-dir', None, 'the directory into which pexes will be written, default: dist.'), + ('pex-args', None, 'additional arguments to the pex tool'), + ] + + boolean_options = [ + 'bdist-all', + ] + + def initialize_options(self): + self.bdist_all = False + self.bdist_dir = None + self.pex_args = '' + + def finalize_options(self): + self.pex_args = self.pex_args.split() + + def _write(self, pex_builder, name, script=None): + builder = pex_builder.clone() + + if script is not None: + builder.set_script(script) + + target = os.path.join(self.bdist_dir, name + '.pex') + + builder.build(target) + + def run(self): + name = self.distribution.get_name() + parser, options_builder = configure_clp() + package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(self.distribution.script_name))) + + if self.bdist_dir is None: + self.bdist_dir = os.path.join(package_dir, 'dist') + + options, reqs = parser.parse_args(self.pex_args) + + if options.entry_point or options.script: + die('Must not specify entry_point or script to --pex-args') + + reqs = [package_dir] + reqs + pex_builder = build_pex(reqs, options, options_builder) + + if self.bdist_all: + for entry_point in self.distribution.entry_points['console_scripts']: + script_name = entry_point.split('=')[0].strip() + log.info('Writing %s to %s.pex' % (script_name, script_name)) + self._write(pex_builder, script_name, script=script_name) + else: + if len(self.distribution.entry_points['console_scripts']) == 1: + script_name = self.distribution.entry_points['console_scripts'][0].split('=')[0].strip() + log.info('Writing %s to %s.pex' % (script_name, name)) + self._write(pex_builder, name, script=script_name) + else: + log.info('Writing environment pex into %s.pex' % name) + self._write(pex_builder, name, script=None) diff --git a/pex/installer.py b/pex/installer.py index 17f497055..acb0bf0c6 100644 --- a/pex/installer.py +++ b/pex/installer.py @@ -190,7 +190,7 @@ def distribution(self): class DistributionPackager(InstallerBase): def mixins(self): mixins = super(DistributionPackager, self).mixins().copy() - mixins.update(setuptools='setuptools>=1') + mixins.update(setuptools=SETUPTOOLS_REQUIREMENT) return mixins def find_distribution(self): diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 5410fc95b..064112554 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -110,8 +110,13 @@ def clone(self, into=None): interpreter exit. """ chroot_clone = self._chroot.clone(into=into) - return self.__class__( - chroot=chroot_clone, interpreter=self._interpreter, pex_info=self._pex_info.copy()) + clone = self.__class__( + chroot=chroot_clone, + interpreter=self._interpreter, + pex_info=self._pex_info.copy()) + for dist in self._distributions: + clone.add_distribution(dist) + return clone def path(self): return self.chroot().path() @@ -204,7 +209,9 @@ def set_script(self, script): self._pex_info.script = script return - raise self.InvalidExecutableSpecification('Could not find script %s in PEX!' % script) + raise self.InvalidExecutableSpecification( + 'Could not find script %r in any distribution %s within PEX!' % ( + script, ', '.join(self._distributions))) def set_entry_point(self, entry_point): """Set the entry point of this PEX environment. diff --git a/pex/pex_info.py b/pex/pex_info.py index d4e88242e..83ca1edd2 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -265,7 +265,8 @@ def update(self, other): def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) + pex_info_copy['distributions'] = self._distributions.copy() return json.dumps(pex_info_copy, **kwargs) def copy(self): - return PexInfo(info=self._pex_info.copy()) + return PexInfo.from_json(self.dump()) diff --git a/setup.py b/setup.py index b50c313e2..95efce434 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ packages = [ 'pex', 'pex.bin', + 'pex.commands', ], install_requires = [ SETUPTOOLS_REQUIREMENT, @@ -52,6 +53,9 @@ 'pytest', ], entry_points = { + 'distutils.commands': [ + 'bdist_pex = pex.commands.bdist_pex:bdist_pex', + ], 'console_scripts': [ 'pex = pex.bin.pex:main', ], From d774cf3a3a1eadf72528afe3cd7c65df1a03984e Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Thu, 7 May 2015 10:56:58 -0700 Subject: [PATCH 15/20] Allows --pex-args to take an argument Prevents this error: $ python setup.py bdist_pex --pex-args='-r gunicorn-req.txt -r ipython-req.txt' usage: setup.py [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] or: setup.py --help [cmd1 cmd2 ...] or: setup.py --help-commands or: setup.py cmd --help error: option --pex-args must not have an argument --- pex/commands/bdist_pex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index f61c96512..d26ecadce 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -14,7 +14,7 @@ class bdist_pex(Command): user_options = [ ('bdist-all', None, 'pexify all defined entry points'), ('bdist-dir', None, 'the directory into which pexes will be written, default: dist.'), - ('pex-args', None, 'additional arguments to the pex tool'), + ('pex-args=', None, 'additional arguments to the pex tool'), ] boolean_options = [ From a53fb0ff51112d2336a0da473864f269bbcb129f Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Thu, 7 May 2015 11:06:54 -0700 Subject: [PATCH 16/20] Don't choke if pkg has no console_scripts Prevents this error: $ python setup.py bdist_pex --pex-args="--index-url=http://packages.corp.surveymonkey.com/index" running bdist_pex Traceback (most recent call last): File "setup.py", line 5, in pbr=True File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/core.py", line 151, in setup dist.run_commands() File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/dist.py", line 953, in run_commands self.run_command(cmd) File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/dist.py", line 972, in run_command cmd_obj.run() File "/Users/marca/dev/git-repos/pex/pex/commands/bdist_pex.py", line 64, in run if len(self.distribution.entry_points['console_scripts']) == 1: KeyError: 'console_scripts' --- pex/commands/bdist_pex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index d26ecadce..31e69d4a4 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -61,7 +61,7 @@ def run(self): log.info('Writing %s to %s.pex' % (script_name, script_name)) self._write(pex_builder, script_name, script=script_name) else: - if len(self.distribution.entry_points['console_scripts']) == 1: + if len(self.distribution.entry_points.get('console_scripts', [])) == 1: script_name = self.distribution.entry_points['console_scripts'][0].split('=')[0].strip() log.info('Writing %s to %s.pex' % (script_name, name)) self._write(pex_builder, name, script=script_name) From 7bd3e3d2bf86e017458e40392d0127bf79bb28ae Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Thu, 7 May 2015 11:26:29 -0700 Subject: [PATCH 17/20] bdist_pex: Nicer output filename 1. Include version number in output filename, like `sdist` and `bdist_wheel` do. 2. Output the actual filename that we are writing to, including the `bdist_dir`. Instead of this: $ python setup.py bdist_pex --pex-args="--index-url=http:/myindex" running bdist_pex Writing environment pex into dummysvc.pex we get this: $ python setup.py bdist_pex --pex-args="--index-url=http://myindex" running bdist_pex Writing environment pex into /Users/marca/dev/surveymonkey/DummySvc/dist/dummysvc-0.0.4.dev53.pex --- pex/commands/bdist_pex.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index 31e69d4a4..5d6843371 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -29,18 +29,17 @@ def initialize_options(self): def finalize_options(self): self.pex_args = self.pex_args.split() - def _write(self, pex_builder, name, script=None): + def _write(self, pex_builder, target, script=None): builder = pex_builder.clone() if script is not None: builder.set_script(script) - target = os.path.join(self.bdist_dir, name + '.pex') - builder.build(target) def run(self): name = self.distribution.get_name() + version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(self.distribution.script_name))) @@ -58,13 +57,15 @@ def run(self): if self.bdist_all: for entry_point in self.distribution.entry_points['console_scripts']: script_name = entry_point.split('=')[0].strip() - log.info('Writing %s to %s.pex' % (script_name, script_name)) - self._write(pex_builder, script_name, script=script_name) + target = os.path.join(self.bdist_dir, script_name + '.pex') + log.info('Writing %s to %s' % (script_name, target)) + self._write(pex_builder, target, script=script_name) else: + target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if len(self.distribution.entry_points.get('console_scripts', [])) == 1: script_name = self.distribution.entry_points['console_scripts'][0].split('=')[0].strip() - log.info('Writing %s to %s.pex' % (script_name, name)) - self._write(pex_builder, name, script=script_name) + log.info('Writing %s to %s' % (script_name, target)) + self._write(pex_builder, target, script=script_name) else: - log.info('Writing environment pex into %s.pex' % name) - self._write(pex_builder, name, script=None) + log.info('Writing environment pex into %s' % target) + self._write(pex_builder, target, script=None) From deb6b09ef25d5117d91292fe4b3071fded06405e Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 11 Aug 2015 14:19:50 -0700 Subject: [PATCH 18/20] Add docs, change default behavior to use namesake command as pex. --- CHANGES.rst | 11 ++- docs/buildingpex.rst | 170 ++++++++++++++++++++++++++++++-------- pex/commands/bdist_pex.py | 58 ++++++++----- pex/pex_info.py | 2 +- 4 files changed, 178 insertions(+), 63 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 12b445d47..77d922ca4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ CHANGES 1.1.0.dev0 ---------- +* Adds the ``bdist_pex`` command to setuptools. + `#99 `_. + * Bug fix: We did not normalize package names in ``ResolvableSet``, so it was possible to depend on ``sphinx`` and ``Sphinx-1.4a0.tar.gz`` and get two versions build and included into the pex. `#147 `_. @@ -16,22 +19,22 @@ CHANGES 1.0.3 ----- -* Bug fix: Accommodate OSX `Python` python binaries. Previously the OSX python distributions shipped +* Bug fix: Accommodate OSX ``Python`` python binaries. Previously the OSX python distributions shipped with OSX, XCode and available via https://www.python.org/downloads/ could fail to be detected using - the `PythonInterpreter` class. + the ``PythonInterpreter`` class. Fixes `#144 `_. * Bug fix: PEX_SCRIPT failed when the script was from a not-zip-safe egg. Original PR `#139 `_. -* Bug fix: `sys.exit` called without arguments would cause `None` to be printed on stderr since pex 1.0.1. +* Bug fix: ``sys.exit`` called without arguments would cause `None` to be printed on stderr since pex 1.0.1. `#143 `_. ----- 1.0.2 ----- -* Bug fix: PEX-INFO values were overridden by environment `Variables` with default values that were +* Bug fix: PEX-INFO values were overridden by environment ``Variables`` with default values that were not explicitly set in the environment. Fixes `#135 `_. diff --git a/docs/buildingpex.rst b/docs/buildingpex.rst index 8a16acfee..dcbf19915 100644 --- a/docs/buildingpex.rst +++ b/docs/buildingpex.rst @@ -1,12 +1,13 @@ .. _buildingpex: -******************* Building .pex files -******************* +=================== The easiest way to build .pex files is with the ``pex`` utility, which is made available when you ``pip install pex``. Do this within a virtualenv, then you can use -pex to bootstrap itself:: +pex to bootstrap itself: + +.. code-block:: bash $ pex pex requests -c pex -o ~/bin/pex @@ -15,12 +16,25 @@ console script named "pex", saving it in ~/bin/pex. At this point, assuming ~/bin is on your $PATH, then you can use pex in or outside of any virtualenv. +The second easiest way to build .pex files is using the ``bdist_pex`` setuptools command +which is available if you ``pip install pex``. For example, to clone and build pip from source: + +.. code-block:: bash + + $ git clone https://github.com/pypa/pip && cd pip + $ python setup.py bdist_pex + running bdist_pex + Writing pip to dist/pip-7.2.0.dev0.pex + +Both are described in more detail below. Invoking the ``pex`` utility ----------------------------- +============================ The ``pex`` utility has no required arguments and by default will construct an empty environment -and invoke it. When no entry point is specified, "invocation" means starting an interpreter:: +and invoke it. When no entry point is specified, "invocation" means starting an interpreter: + +.. code-block:: bash $ pex Python 2.6.9 (unknown, Jan 2 2014, 14:52:48) @@ -33,7 +47,9 @@ This creates an ephemeral environment that only exists for the duration of the ` and is garbage collected immediately on exit. You can tailor which interpreter is used by specifying ``--python=PATH``. PATH can be either the -absolute path of a Python binary or the name of a Python interpreter within the environment, e.g.:: +absolute path of a Python binary or the name of a Python interpreter within the environment, e.g.: + +.. code-block:: bash $ pex --python=python3.3 Python 3.3.3 (default, Jan 2 2014, 14:57:01) @@ -53,7 +69,9 @@ Specifying requirements Requirements are specified using the same form as expected by ``pip`` and ``setuptools``, e.g. ``flask``, ``setuptools==2.1.2``, ``Django>=1.4,<1.6``. These are specified as arguments to pex and any number (including 0) may be specified. For example, to start an environment with ``flask`` -and ``psutil>1``:: +and ``psutil>1``: + +.. code-block:: bash $ pex flask 'psutil>1' Python 2.6.9 (unknown, Jan 2 2014, 14:52:48) @@ -62,14 +80,18 @@ and ``psutil>1``:: (InteractiveConsole) >>> -You can then import and manipulate modules like you would otherwise:: +You can then import and manipulate modules like you would otherwise: + +.. code-block:: bash >>> import flask >>> import psutil >>> ... Requirements can also be specified using the requirements.txt format, using ``pex -r``. This can be a handy -way to freeze a virtualenv into a PEX file:: +way to freeze a virtualenv into a PEX file: + +.. code-block:: bash $ pex -r <(pip freeze) -o my_application.pex @@ -80,10 +102,12 @@ Specifying entry points Entry points define how the environment is executed and may be specified in one of three ways. pex -- script.py -^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~ As mentioned above, if no entry points are specified, the default behavior is to emulate an -interpreter. First we create a simple flask application:: +interpreter. First we create a simple flask application: + +.. code-block:: bash $ cat < flask_hello_world.py > from flask import Flask @@ -96,17 +120,21 @@ interpreter. First we create a simple flask application:: > app.run() > EOF -Then, like an interpreter, if a source file is specified as a parameter to pex, it is invoked:: +Then, like an interpreter, if a source file is specified as a parameter to pex, it is invoked: + +.. code-block:: bash $ pex flask -- ./flask_hello_world.py * Running on http://127.0.0.1:5000/ pex -m -^^^^^^ +~~~~~~ Your code may be within the PEX file or it may be some predetermined entry point within the standard library. ``pex -m`` behaves very similarly to ``python -m``. Consider -``python -m pydoc``:: +``python -m pydoc``: + +.. code-block:: bash $ python -m pydoc pydoc - the Python documentation tool @@ -117,7 +145,9 @@ within the standard library. ``pex -m`` behaves very similarly to ``python -m`` reference to a class or function within a module or module in a ... -This can be emulated using the ``pex`` tool using ``-m pydoc``:: +This can be emulated using the ``pex`` tool using ``-m pydoc``: + +.. code-block:: bash $ pex -m pydoc pydoc - the Python documentation tool @@ -129,7 +159,9 @@ This can be emulated using the ``pex`` tool using ``-m pydoc``:: ... Arguments will be passed unescaped following ``--`` on the command line. So in order to -get pydoc help on the ``flask.app`` package in Flask:: +get pydoc help on the ``flask.app`` package in Flask: + +.. code-block:: bash $ pex flask -m pydoc -- flask.app @@ -151,7 +183,9 @@ Entry points can also take the form ``package:target``, such as ``sphinx:main`` and Fabric respectively. This is roughly equivalent to running a script that does ``from package import target; target()``. This can be a powerful way to invoke Python applications without ever having to ``pip install`` -anything, for example a one-off invocation of Sphinx with the readthedocs theme available:: +anything, for example a one-off invocation of Sphinx with the readthedocs theme available: + +.. code-block:: bash $ pex sphinx sphinx_rtd_theme -e sphinx:main -- --help Sphinx v1.2.2 @@ -165,11 +199,13 @@ anything, for example a one-off invocation of Sphinx with the readthedocs theme ... pex -c -^^^^^^ +~~~~~~ If you don't know the ``package:target`` for the console scripts of your favorite python packages, pex allows you to use ``-c`` to specify a console script as defined -by the distribution. For example, Fabric provides the ``fab`` tool when pip installed:: +by the distribution. For example, Fabric provides the ``fab`` tool when pip installed: + +.. code-block:: bash $ pex Fabric -c fab -- --help Fatal error: Couldn't find any fabfiles! @@ -178,7 +214,9 @@ by the distribution. For example, Fabric provides the ``fab`` tool when pip ins Aborting. -Even scripts defined by the "scripts" section of a distribution can be used, e.g. with boto:: +Even scripts defined by the "scripts" section of a distribution can be used, e.g. with boto: + +.. code-block:: bash $ pex boto -c mturk usage: mturk [-h] [-P] [--nicknames PATH] @@ -194,16 +232,22 @@ Each of the commands above have been manipulating ephemeral PEX environments -- exist for the duration of the pex command lifetime and immediately garbage collected. If the ``-o PATH`` option is specified, a PEX file of the environment is saved to disk at ``PATH``. For example -we can package a standalone Sphinx as above:: +we can package a standalone Sphinx as above: + +.. code-block:: bash $ pex sphinx sphinx_rtd_theme -c sphinx -o sphinx.pex -Instead of executing the environment, it is saved to disk:: +Instead of executing the environment, it is saved to disk: + +.. code-block:: bash $ ls -l sphinx.pex -rwxr-xr-x 1 wickman wheel 4988494 Mar 11 17:48 sphinx.pex -This is an executable environment and can be executed as before:: +This is an executable environment and can be executed as before: + +.. code-block:: bash $ ./sphinx.pex --help Sphinx v1.2.2 @@ -219,16 +263,22 @@ This is an executable environment and can be executed as before:: As before, entry points are not required, and if not specified the PEX will default to just dropping into an interpreter. If an alternate interpreter is specified with ``--python``, e.g. pypy, it will be the -default hashbang in the PEX file:: +default hashbang in the PEX file: + +.. code-block:: bash $ pex --python=pypy flask -o flask-pypy.pex -The hashbang of the PEX file specifies PyPy:: +The hashbang of the PEX file specifies PyPy: + +.. code-block:: bash $ head -1 flask-pypy.pex #!/usr/bin/env pypy -and when invoked uses the environment PyPy:: +and when invoked uses the environment PyPy: + +.. code-block:: bash $ ./flask-pypy.pex Python 2.7.3 (87aa9de10f9c, Nov 24 2013, 20:57:21) @@ -238,7 +288,9 @@ and when invoked uses the environment PyPy:: >>> import flask To specify an explicit Python shebang line (e.g. from a non-standard location or not on $PATH), -you can use the ``--python-shebang`` option:: +you can use the ``--python-shebang`` option: + +.. code-block:: bash $ dist/pex --python-shebang='/Users/wickman/Python/CPython-3.4.2/bin/python3.4' -o my.pex $ head -1 my.pex @@ -253,11 +305,13 @@ Tailoring requirement resolution In general, ``pex`` honors the same options as pip when it comes to resolving packages. Like pip, by default ``pex`` fetches artifacts from PyPI. This can be disabled with ``--no-index``. -If PyPI fetching is disabled, you will need to specify a search repository via ``-f/--find-links``. +If PyPI fetching is disabled, you will need to specify a search repository via ``-f/--find-links``. This may be a directory on disk or a remote simple http server. For example, you can delegate artifact fetching and resolution to ``pip wheel`` for whatever -reason -- perhaps you're running a firewalled mirror -- but continue to package with pex:: +reason -- perhaps you're running a firewalled mirror -- but continue to package with pex: + +.. code-block:: bash $ pip wheel -w /tmp/wheelhouse sphinx sphinx_rtd_theme $ pex -f /tmp/wheelhouse --no-index -e sphinx:main -o sphinx.pex sphinx sphinx_rtd_theme @@ -272,9 +326,9 @@ that can be used to override the runtime behavior. ``--zip-safe``/``--not-zip-safe`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Whether or not to treat the environment as zip-safe. By default PEX files are listed as zip safe. +Whether or not to treat the environment as zip-safe. By default PEX files are listed as zip safe. If ``--not-zip-safe`` is specified, the source of the PEX will be written to disk prior to invocation rather than imported via the zipimporter. NOTE: Distribution zip-safe bits will still be honored even if the PEX is marked as zip-safe. For example, included .eggs may be marked as @@ -283,14 +337,14 @@ and written to disk prior to PEX invocation. ``--not-zip-safe`` forces ``--alwa ``--always-write-cache`` -^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~ Always write all packaged dependencies within the PEX to disk prior to invocation. This forces the zip-safe bit of any dependency to be ignored. ``--inherit-path`` -^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~ By default, PEX environments are completely scrubbed empty of any packages installed on the global site path. Setting ``--inherit-path`` allows packages within site-packages to be considered as candidate distributions @@ -300,7 +354,7 @@ if a package does not package correctly an an egg or wheel.) ``--ignore-errors`` -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~ If not all of the PEX environment's dependencies resolve correctly (e.g. you are overriding the current Python interpreter with ``PEX_PYTHON``) this forces the PEX file to execute despite this. Can be useful @@ -308,7 +362,7 @@ in certain situations when particular extensions may not be necessary to run a p ``--platform`` -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~ The platform to build the pex for. Right now it defaults to the current system, but you can specify something like ``linux-x86_64`` or ``macosx-10.6-x86_64``. This will look for bdists for the particular platform. @@ -322,8 +376,52 @@ The source of truth for these environment variables can be found in the `pex.variables API `_. +Using ``bdist_pex`` +=================== + +pex provides a convenience command for use in setuptools. ``python setup.py +bdist_pex`` is a simple way to build executables for Python projects that +adhere to standard naming conventions. + +``bdist_pex`` +------------- + +The default behavior of ``bdist_pex`` is to build an executable using the +console script of the same name as the package. For example, pip has three +entry points: ``pip``, ``pip2`` and ``pip2.7`` if you're using Python 2.7. Since +there exists an entry point named ``pip`` in the ``console_scripts`` section +of the entry points, that entry point is chosen and an executable pex is produced. The pex file +will have the version number appended, e.g. ``pip-7.2.0.pex``. + +If no console scripts are provided, or the only console scripts available do +not bear the same name as the package, then an environment pex will be +produced. An environment pex is a pex file that drops you into an +interpreter with all necessary dependencies but stops short of invoking a +specific module or function. + +``bdist_pex --bdist-all`` +------------------------- + +If you would like to build all the console scripts defined in the package instead of +just the namesake script, ``--bdist-all`` will write all defined entry_points but omit +version numbers and the ``.pex`` suffix. This can be useful if you would like to +virtually install a Python package somewhere on your ``$PATH`` without doing something +scary like ``sudo pip install``: + +.. code-block:: bash + + $ git clone https://github.com/sphinx-doc/sphinx && cd sphinx + $ python setup.py bist_pex --bdist-all --bdist-dir=$HOME/bin + running bdist_pex + Writing sphinx-apidoc to /Users/wickman/bin/sphinx-apidoc + Writing sphinx-build to /Users/wickman/bin/sphinx-build + Writing sphinx-quickstart to /Users/wickman/bin/sphinx-quickstart + Writing sphinx-autogen to /Users/wickman/bin/sphinx-autogen + $ sphinx-apidoc --help | head -1 + Usage: sphinx-apidoc [options] -o [exclude_path, ...] + Other ways to build PEX files ------------------------------ +============================= There are other supported ways to build pex files: * Using pants. See `Pants Python documentation `_. diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index 5d6843371..4f7950fee 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -1,23 +1,24 @@ import os -import pprint from distutils import log -from pex.bin.pex import configure_clp, build_pex -from pex.common import die - from setuptools import Command +from pex.bin.pex import build_pex, configure_clp +from pex.common import die +from pex.variables import ENV + -class bdist_pex(Command): - description = "create a PEX file from a source distribution" +# Suppress checkstyle violations due to setuptools command requirements. +class bdist_pex(Command): # noqa + description = "create a PEX file from a source distribution" # noqa - user_options = [ + user_options = [ # noqa ('bdist-all', None, 'pexify all defined entry points'), - ('bdist-dir', None, 'the directory into which pexes will be written, default: dist.'), + ('bdist-dir=', None, 'the directory into which pexes will be written, default: dist.'), ('pex-args=', None, 'additional arguments to the pex tool'), ] - boolean_options = [ + boolean_options = [ # noqa 'bdist-all', ] @@ -41,7 +42,8 @@ def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() - package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(self.distribution.script_name))) + package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( + self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') @@ -52,20 +54,32 @@ def run(self): die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs - pex_builder = build_pex(reqs, options, options_builder) + + with ENV.patch(PEX_VERBOSE=str(options.verbosity)): + pex_builder = build_pex(reqs, options, options_builder) + + def split_and_strip(entry_point): + console_script, entry_point = entry_point.split('=', 2) + return console_script.strip(), entry_point.strip() + + try: + console_scripts = dict(split_and_strip(script) + for script in self.distribution.entry_points.get('console_scripts', [])) + except ValueError: + console_scripts = {} if self.bdist_all: - for entry_point in self.distribution.entry_points['console_scripts']: - script_name = entry_point.split('=')[0].strip() - target = os.path.join(self.bdist_dir, script_name + '.pex') + # Write all entry points into unversioned pex files. + for script_name in console_scripts: + target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) - else: + elif name in console_scripts: + # The package has a namesake entry point, so use it. target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') - if len(self.distribution.entry_points.get('console_scripts', [])) == 1: - script_name = self.distribution.entry_points['console_scripts'][0].split('=')[0].strip() - log.info('Writing %s to %s' % (script_name, target)) - self._write(pex_builder, target, script=script_name) - else: - log.info('Writing environment pex into %s' % target) - self._write(pex_builder, target, script=None) + log.info('Writing %s to %s' % (name, target)) + self._write(pex_builder, target, script=name) + else: + # The package has no namesake entry point, so build an environment pex. + log.info('Writing environment pex into %s' % target) + self._write(pex_builder, target, script=None) diff --git a/pex/pex_info.py b/pex/pex_info.py index 83ca1edd2..4cce1eb9d 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -269,4 +269,4 @@ def dump(self, **kwargs): return json.dumps(pex_info_copy, **kwargs) def copy(self): - return PexInfo.from_json(self.dump()) + return self.from_json(self.dump()) From a6dfdfbbc4c575e834cb4ed4020c4eac73248acb Mon Sep 17 00:00:00 2001 From: James Broadhead Date: Mon, 21 Sep 2015 10:55:03 +0100 Subject: [PATCH 19/20] [docs] update header in index.rst - This should stop the top Google link reading 'TLDR' --- docs/index.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 065f4dd6a..344ae0905 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,19 +1,22 @@ ****** -tl;dr +pex ****** +This project is the home of the .pex file, and the ``pex`` tool which can create them. +``pex`` also provides a general purpose Python environment-virtualization solution similar to `virtualenv `_. +pex is short for "Python Executable" -To quickly get started building .pex (PEX) files, go straight to :ref:`buildingpex`. +in brief +=== +To quickly get started building .pex files, go straight to :ref:`buildingpex`. New to python packaging? Check out `packaging.python.org `_. -pex +intro & history === - -pex contains the Python packaging and distribution libraries originally available through the +pex contains the Python packaging and distribution libraries originally available through the `twitter commons `_ but since split out into a separate project. The most notable components of pex are the .pex (Python EXecutable) format and the associated ``pex`` tool which provide a general purpose Python environment virtualization -solution similar in spirit to `virtualenv `_. PEX files have been used by Twitter -to deploy Python applications to production since 2011. +solution similar in spirit to `virtualenv `_. PEX files have been used by Twitter to deploy Python applications to production since 2011. To learn more about what the .pex format is and why it could be useful for you, see :ref:`whatispex` For the impatient, there is also a (slightly outdated) lightning From 62f592d0122753844cd94901deffea75c2df3ee0 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 7 Oct 2015 12:35:18 -0600 Subject: [PATCH 20/20] Migrate to the new travis-ci infra. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c2b2c54e..128804cf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,9 @@ +# Enables support for a docker container-based build +# which should provide faster startup times and beefier +# "machines". +# See: http://docs.travis-ci.com/user/workers/container-based-infrastructure/ +sudo: false + language: python python: 2.7 env: