From 722d2255840f9809f809185b0db60520a22d10c5 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Tue, 1 Dec 2015 10:44:54 -0800 Subject: [PATCH 01/13] Add the ability to specify pex_root on command line * add pex_root option * fix some places where ~/.pex was hard coded --- pex/bin/pex.py | 27 +++++++++++++++++++++++---- pex/commands/bdist_pex.py | 2 +- pex/http.py | 2 +- pex/pex_info.py | 2 +- pex/variables.py | 2 +- 5 files changed, 27 insertions(+), 8 deletions(-) mode change 100644 => 100755 pex/bin/pex.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py old mode 100644 new mode 100755 index af570490e..388075709 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -159,9 +159,9 @@ def configure_clp_pex_resolution(parser, builder): group.add_option( '--cache-dir', dest='cache_dir', - default=os.path.expanduser('~/.pex/build'), + default='{pex_root}/build', help='The local cache directory to use for speeding up requirement ' - 'lookups. [Default: %default]') + 'lookups. [Default: ~/.pex/build]') group.add_option( '--cache-ttl', @@ -266,9 +266,9 @@ def configure_clp_pex_environment(parser): group.add_option( '--interpreter-cache-dir', dest='interpreter_cache_dir', - default=os.path.expanduser('~/.pex/interpreters'), + default='{pex_root}/interpreters', help='The interpreter cache to use for keeping track of interpreter dependencies ' - 'for the pex tool. [Default: %default]') + 'for the pex tool. [Default: ~/.pex/interpreters]') parser.add_option_group(group) @@ -337,6 +337,13 @@ def configure_clp(): callback=increment_verbosity, help='Turn on logging verbosity, may be specified multiple times.') + parser.add_option( + '--pex-root', + dest='pex_root', + default=None, + help='Specify the pex root used in this invocation of pex. [Default: ~/.pex]' + ) + parser.add_option( '--help-variables', action='callback', @@ -491,6 +498,11 @@ def build_pex(args, options, resolver_option_builder): return pex_builder +def make_relative_to_root(path): + """Update options so that defaults are user relative to specified pex_root.""" + return os.path.normpath(path.format(pex_root=ENV.PEX_ROOT)) + + def main(): parser, resolver_options_builder = configure_clp() @@ -503,6 +515,13 @@ def main(): args, cmdline = args, [] options, reqs = parser.parse_args(args=args) + if options.pex_root: + ENV.set('PEX_ROOT', options.pex_root) + else: + options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. + + options.cache_dir = make_relative_to_root(options.cache_dir) + options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index 7a261f298..152182e6f 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -55,7 +55,7 @@ def run(self): reqs = [package_dir] + reqs - with ENV.patch(PEX_VERBOSE=str(options.verbosity)): + with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) def split_and_strip(entry_point): diff --git a/pex/http.py b/pex/http.py index 5b91a5c7b..ac7001eaf 100644 --- a/pex/http.py +++ b/pex/http.py @@ -248,7 +248,7 @@ def content(self, link): class CachedRequestsContext(RequestsContext): """A requests-based Context with CacheControl support.""" - DEFAULT_CACHE = '~/.pex/cache' + DEFAULT_CACHE = os.path.join(ENV.PEX_ROOT, 'cache') def __init__(self, cache=None, **kw): self._cache = os.path.realpath(os.path.expanduser(cache or self.DEFAULT_CACHE)) diff --git a/pex/pex_info.py b/pex/pex_info.py index 4cce1eb9d..9355e7c4b 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -30,7 +30,7 @@ class PexInfo(object): requirements: list # list of requirements for this environment # Environment options - pex_root: ~/.pex # root of all pex-related files + pex_root: string # root of all pex-related files eg: ~/.pex entry_point: string # entry point into this pex script: string # script to execute in this pex environment # at most one of script/entry_point can be specified diff --git a/pex/variables.py b/pex/variables.py index 5075f368b..38208754d 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -238,7 +238,7 @@ def PEX_ROOT(self): The directory location for PEX to cache any dependencies and code. PEX must write not-zip-safe eggs and all wheels to disk in order to activate them. Default: ~/.pex """ - return self._get_path('PEX_ROOT', default=None) + return self._get_path('PEX_ROOT', default=os.path.expanduser('~/.pex')) @property def PEX_PATH(self): From 611155855e5cec147d48b8857bdac86407cfe378 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 11:30:42 -0700 Subject: [PATCH 02/13] add test --- tests/test_pex_info.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_pex_info.py b/tests/test_pex_info.py index 0e6a01a7f..d7f54110e 100644 --- a/tests/test_pex_info.py +++ b/tests/test_pex_info.py @@ -7,7 +7,8 @@ from pex.orderedset import OrderedSet from pex.pex_info import PexInfo -from pex.variables import Variables +from pex.variables import Variables, ENV +from pex.bin.pex import make_relative_to_root def make_pex_info(requirements): @@ -47,6 +48,14 @@ def test_from_empty_env(): assert_same_info(PexInfo(info=info), PexInfo.from_env(env=environ)) +def test_make_relative(): + with ENV.patch(PEX_ROOT='/pex_root'): + assert '/pex_root/interpreters' == make_relative_to_root('{pex_root}/interpreters') + + #Verify the user can specify arbitrary absolute paths. + assert '/tmp/interpreters' == make_relative_to_root('/tmp/interpreters') + + def test_from_env(): pex_root = os.path.realpath('/pex_root') environ = dict(PEX_ROOT=pex_root, From 71be0402bf229bc38a1b791a77cc8993f655d07e Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 12:47:07 -0700 Subject: [PATCH 03/13] fix import order --- tests/test_pex_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pex_info.py b/tests/test_pex_info.py index d7f54110e..1bf2ff65a 100644 --- a/tests/test_pex_info.py +++ b/tests/test_pex_info.py @@ -5,10 +5,10 @@ import pytest +from pex.bin.pex import make_relative_to_root from pex.orderedset import OrderedSet from pex.pex_info import PexInfo from pex.variables import Variables, ENV -from pex.bin.pex import make_relative_to_root def make_pex_info(requirements): From 8fa60141c79d3ad41a4e25781c0496b6dda1e6d7 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 12:54:25 -0700 Subject: [PATCH 04/13] fix import order --- tests/test_pex_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pex_info.py b/tests/test_pex_info.py index 1bf2ff65a..63634fe36 100644 --- a/tests/test_pex_info.py +++ b/tests/test_pex_info.py @@ -8,7 +8,7 @@ from pex.bin.pex import make_relative_to_root from pex.orderedset import OrderedSet from pex.pex_info import PexInfo -from pex.variables import Variables, ENV +from pex.variables import ENV, Variables def make_pex_info(requirements): From 35bd2387c131062e3a6dfb27fa26553985b0ea9e Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 16:25:31 -0700 Subject: [PATCH 05/13] add integration test --- pex/bin/pex.py | 15 +++++++++++---- pex/testing.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_integration.py | 19 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 388075709..0eae60763 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -503,11 +503,18 @@ def make_relative_to_root(path): return os.path.normpath(path.format(pex_root=ENV.PEX_ROOT)) -def main(): +def main(args=None, log_callback=None): + if log_callback is not None: + log = log_callback + parser, resolver_options_builder = configure_clp() - # split arguments early because optparse is dumb - args = sys.argv[1:] + if args is None: + # split arguments early because optparse is dumb + args = sys.argv[1:] + else: + args = args[:] + try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] @@ -542,7 +549,7 @@ def main(): log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) - sys.exit(pex.run(args=list(cmdline))) + raise SystemExit(pex.run(args=list(cmdline))) if __name__ == '__main__': diff --git a/pex/testing.py b/pex/testing.py index 2726486f7..402294dc3 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -8,8 +8,10 @@ import sys import tempfile import zipfile +from collections import namedtuple from textwrap import dedent +from .bin.pex import main from .common import safe_mkdir, safe_rmtree from .compatibility import nested from .installer import EggInstaller, Packager @@ -173,6 +175,41 @@ def write_simple_pex(td, exe_contents, dists=None, coverage=False): return pb +class integ_results(namedtuple('results', 'output return_code exception')): + """Convenience object to return integration run results.""" + def assert_success(self): + assert self.exception is None and self.return_code is None + + def assert_failure(self): + assert self.exception or self.return_code + + +def run_pex_command(args, env=None): + """Simulate running pex command for integration testing. + + This is different from run_simple_pex in that it calls the pex command rather + than running a generated pex. This is useful for testing end to end runs + with specific command line arguments or env options. + """ + def logger_callback(_output): + def mock_logger(msg, v=None): + _output.append(msg) + + return mock_logger + + exception = None + error_code = None + output = [] + + try: + main(args=args, log_callback=logger_callback(output)) + except SystemExit as e: + error_code = e.code + except Exception as e: + exception = e + return integ_results(output, error_code, exception) + + # TODO(wickman) Why not PEX.run? def run_simple_pex(pex, args=(), env=None): po = subprocess.Popen( diff --git a/tests/test_integration.py b/tests/test_integration.py index 81386a7bf..5c3ea3bbc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -8,7 +8,7 @@ from twitter.common.contextutil import environment_as, temporary_dir from pex.compatibility import WINDOWS -from pex.testing import run_simple_pex_test +from pex.testing import run_simple_pex_test, run_pex_command from pex.util import named_temporary_file @@ -23,6 +23,23 @@ def test_pex_raise(): run_simple_pex_test(body, coverage=True) +def test_pex_root(): + with temporary_dir() as tmp_home: + with environment_as(HOME=tmp_home): + with temporary_dir() as td: + with temporary_dir() as output_dir: + env = os.environ.copy() + env['PEX_INTERPRETER'] = '1' + + output_path = os.path.join(output_dir, 'pex.pex') + args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={}'.format(td)] + results = run_pex_command(args=args, env=env) + results.assert_success() + assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' + assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' + assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.' + + def test_pex_interpreter(): with named_temporary_file() as fp: fp.write(b"print('Hello world')") From 64aedc9cf28a9bf352c183e1440ca726377a3eb1 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 16:29:35 -0700 Subject: [PATCH 06/13] fix py26 issue --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 5c3ea3bbc..e60914574 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -32,7 +32,7 @@ def test_pex_root(): env['PEX_INTERPRETER'] = '1' output_path = os.path.join(output_dir, 'pex.pex') - args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={}'.format(td)] + args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)] results = run_pex_command(args=args, env=env) results.assert_success() assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' From d10a7e3d3b446108116612169a8a99cd9d763ee5 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 16:31:16 -0700 Subject: [PATCH 07/13] fix sort issue --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index e60914574..98b02a456 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -8,7 +8,7 @@ from twitter.common.contextutil import environment_as, temporary_dir from pex.compatibility import WINDOWS -from pex.testing import run_simple_pex_test, run_pex_command +from pex.testing import run_pex_command, run_simple_pex_test from pex.util import named_temporary_file From f079a2c5b95825dd137df878550f0e241ece53ca Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Wed, 6 Apr 2016 16:38:28 -0700 Subject: [PATCH 08/13] fix style issues --- pex/testing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pex/testing.py b/pex/testing.py index 402294dc3..84acd0779 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -175,8 +175,9 @@ def write_simple_pex(td, exe_contents, dists=None, coverage=False): return pb -class integ_results(namedtuple('results', 'output return_code exception')): +class IntegResults(namedtuple('results', 'output return_code exception')): """Convenience object to return integration run results.""" + def assert_success(self): assert self.exception is None and self.return_code is None @@ -207,7 +208,7 @@ def mock_logger(msg, v=None): error_code = e.code except Exception as e: exception = e - return integ_results(output, error_code, exception) + return IntegResults(output, error_code, exception) # TODO(wickman) Why not PEX.run? From 2b5890171522396acfda4ae642319fc5e2c1c1bb Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Thu, 7 Apr 2016 11:22:37 -0700 Subject: [PATCH 09/13] update defaults --- pex/bin/pex.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 0eae60763..c7a3c40b5 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -503,18 +503,9 @@ def make_relative_to_root(path): return os.path.normpath(path.format(pex_root=ENV.PEX_ROOT)) -def main(args=None, log_callback=None): - if log_callback is not None: - log = log_callback - +def main(args=None, log_callback=log): parser, resolver_options_builder = configure_clp() - if args is None: - # split arguments early because optparse is dumb - args = sys.argv[1:] - else: - args = args[:] - try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] @@ -553,4 +544,5 @@ def main(args=None, log_callback=None): if __name__ == '__main__': - main() + # split arguments early because optparse is dumb + main(sys.argv[1:]) From 6538ff66458fbd5246d314989d6d80b6d676bb1a Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Thu, 7 Apr 2016 12:44:49 -0700 Subject: [PATCH 10/13] Update logger to allow a different callback. --- pex/bin/pex.py | 18 ++++++++++++++---- pex/testing.py | 6 +++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c7a3c40b5..e034987b2 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -42,10 +42,20 @@ INVALID_ENTRY_POINT = 104 -def log(msg, v=False): - if v: - print(msg, file=sys.stderr) +class logger(object): + def _default_logger(self, msg, v): + if v: + print(msg, file=sys.stderr) + _logger = _default_logger + + def __call__(self, msg, v): + self._logger(msg, v) + + def set_logger(self, logger_callback): + self._logger = logger_callback + +log = logger() def parse_bool(option, opt_str, _, parser): setattr(parser.values, option.dest, not opt_str.startswith('--no')) @@ -503,7 +513,7 @@ def make_relative_to_root(path): return os.path.normpath(path.format(pex_root=ENV.PEX_ROOT)) -def main(args=None, log_callback=log): +def main(args): parser, resolver_options_builder = configure_clp() try: diff --git a/pex/testing.py b/pex/testing.py index 84acd0779..712ae0163 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -11,7 +11,7 @@ from collections import namedtuple from textwrap import dedent -from .bin.pex import main +from .bin.pex import log, main from .common import safe_mkdir, safe_rmtree from .compatibility import nested from .installer import EggInstaller, Packager @@ -201,9 +201,9 @@ def mock_logger(msg, v=None): exception = None error_code = None output = [] - + log.set_logger(logger_callback(output)) try: - main(args=args, log_callback=logger_callback(output)) + main(args=args) except SystemExit as e: error_code = e.code except Exception as e: From f4e4ef1651c38d0758bdd5cd6c6cc69f1e42d221 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Thu, 7 Apr 2016 13:18:17 -0700 Subject: [PATCH 11/13] fix style issue --- pex/bin/pex.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index e034987b2..513c543ff 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -47,16 +47,17 @@ def _default_logger(self, msg, v): if v: print(msg, file=sys.stderr) - _logger = _default_logger + _LOGGER = _default_logger def __call__(self, msg, v): - self._logger(msg, v) + self._LOGGER(msg, v) def set_logger(self, logger_callback): - self._logger = logger_callback + self._LOGGER = logger_callback log = logger() + def parse_bool(option, opt_str, _, parser): setattr(parser.values, option.dest, not opt_str.startswith('--no')) From e56f4397f6355e2736af1dce894a1a1c9b15da71 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Thu, 7 Apr 2016 13:24:07 -0700 Subject: [PATCH 12/13] fix style issue --- pex/bin/pex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 513c543ff..18fdd6512 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -42,7 +42,7 @@ INVALID_ENTRY_POINT = 104 -class logger(object): +class Logger(object): def _default_logger(self, msg, v): if v: print(msg, file=sys.stderr) @@ -55,7 +55,7 @@ def __call__(self, msg, v): def set_logger(self, logger_callback): self._LOGGER = logger_callback -log = logger() +log = Logger() def parse_bool(option, opt_str, _, parser): From 556a30a9a2f795417fdcac62b4e67f3cb08e4e06 Mon Sep 17 00:00:00 2001 From: Matt Olsen Date: Tue, 26 Apr 2016 14:53:34 -0700 Subject: [PATCH 13/13] use sys.exit rather than SystemExit --- pex/bin/pex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 18fdd6512..a60bdc845 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -551,7 +551,7 @@ def main(args): log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) - raise SystemExit(pex.run(args=list(cmdline))) + sys.exit(pex.run(args=list(cmdline))) if __name__ == '__main__':