From 5ef7084c067bbf6d57a482d0ed435f0ec638870a Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 26 Oct 2018 16:36:36 -0600 Subject: [PATCH] Vendor setuptools and wheel. Fixes #607 --- MANIFEST.in | 1 + pex/bin/pex.py | 107 +++----------------- pex/commands/bdist_pex.py | 4 +- pex/compiler.py | 2 +- pex/environment.py | 8 +- pex/executor.py | 24 ++--- pex/finders.py | 18 ---- pex/installer.py | 87 +--------------- pex/interpreter.py | 78 ++++----------- pex/package.py | 4 - pex/pex.py | 16 +-- pex/pex_builder.py | 8 +- pex/requirements.py | 17 +++- pex/resolvable.py | 50 +++++----- pex/resolver.py | 4 +- pex/testing.py | 94 ++++++++--------- pex/translator.py | 5 +- pex/vendor/__init__.py | 72 ++++++++++++++ pex/vendor/__main__.py | 43 ++++++++ pex/version.py | 8 +- scripts/coverage.sh | 2 +- scripts/style.sh | 7 ++ setup.py | 63 ++++++------ tests/test_environment.py | 31 ++---- tests/test_executor.py | 22 ++-- tests/test_inherits_path_option.py | 19 ++-- tests/test_installer.py | 30 +++--- tests/test_integration.py | 69 ++++++------- tests/test_interpreter.py | 45 +-------- tests/test_pex.py | 155 +++++++++++++---------------- tests/test_pex_binary.py | 10 +- tests/test_pex_bootstrapper.py | 20 +++- tests/test_pex_builder.py | 48 +++++---- tests/test_resolvable.py | 16 +-- tests/test_resolver.py | 82 +++++++-------- tests/test_util.py | 35 +++---- tox.ini | 40 +++++--- 37 files changed, 577 insertions(+), 767 deletions(-) create mode 100644 pex/vendor/__init__.py create mode 100644 pex/vendor/__main__.py create mode 100755 scripts/style.sh diff --git a/MANIFEST.in b/MANIFEST.in index e1a99fa2f..b9e0e4f35 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,5 @@ include *.py *.rst *.ini MANIFEST.in LICENSE recursive-include docs * recursive-include tests * recursive-include scripts * +recursive-include pex/vendor/_vendored * recursive-exclude * *.pyc *~ diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c4ea84798..b356b54e8 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -2,41 +2,33 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). """ -The pex.pex utility builds PEX environments and .pex files specified by +The pex.bin.pex utility builds PEX environments and .pex files specified by sources, requirements and their dependencies. """ from __future__ import absolute_import, print_function -import functools import os -import shutil import sys from optparse import OptionGroup, OptionParser, OptionValueError from textwrap import TextWrapper -from pex.archiver import Archiver -from pex.base import maybe_requirement -from pex.common import die, safe_delete, safe_mkdir, safe_mkdtemp -from pex.crawler import Crawler +from pex import vendor +from pex.common import die, safe_delete, safe_mkdtemp from pex.fetcher import Fetcher, PyPIFetcher -from pex.http import Context -from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import validate_constraints -from pex.iterator import Iterator -from pex.package import EggPackage, SourcePackage from pex.pex import PEX from pex.pex_bootstrapper import find_compatible_interpreters from pex.pex_builder import PEXBuilder from pex.platforms import Platform from pex.requirements import requirements_from_file -from pex.resolvable import Resolvable +from pex.resolvable import resolvables_from_iterable from pex.resolver import Unsatisfiable, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.tracer import TRACER from pex.variables import ENV, Variables -from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT, __version__ +from pex.version import __version__ CANNOT_DISTILL = 101 CANNOT_SETUP_INTERPRETER = 102 @@ -499,78 +491,6 @@ def _safe_link(src, dst): os.symlink(src, dst) -def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider): - # Short-circuit if there is a local copy - if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)): - egg = EggPackage(os.path.realpath(target_link)) - if egg.satisfies(requirement): - return egg - - context = Context.get() - iterator = Iterator(fetchers=fetchers, crawler=Crawler(context)) - links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)] - - with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2): - for link in links: - with TRACER.timed('Fetching %s' % link, V=3): - sdist = context.fetch(link) - - with TRACER.timed('Installing %s' % link, V=3): - installer = installer_provider(sdist) - dist_location = installer.bdist() - target_location = os.path.join( - os.path.dirname(target_link), os.path.basename(dist_location)) - shutil.move(dist_location, target_location) - _safe_link(target_location, target_link) - - return EggPackage(target_location) - - -def resolve_interpreter(cache, fetchers, interpreter, requirement): - """Resolve an interpreter with a specific requirement. - - Given a :class:`PythonInterpreter` and a requirement, return an - interpreter with the capability of resolving that requirement or - ``None`` if it's not possible to install a suitable requirement.""" - requirement = maybe_requirement(requirement) - - # short circuit - if interpreter.satisfies([requirement]): - return interpreter - - def installer_provider(sdist): - return EggInstaller( - Archiver.unpack(sdist), - strict=requirement.key != 'setuptools', - interpreter=interpreter) - - interpreter_dir = os.path.join(cache, str(interpreter.identity)) - safe_mkdir(interpreter_dir) - - egg = _resolve_and_link_interpreter( - requirement, - fetchers, - os.path.join(interpreter_dir, requirement.key), - installer_provider) - - if egg: - return interpreter.with_extra(egg.name, egg.raw_version, egg.path) - - -def setup_interpreter(interpreter, interpreter_cache_dir, repos, use_wheel): - with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): - resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos) - - # resolve setuptools - interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) - - # possibly resolve wheel - if interpreter and use_wheel: - interpreter = resolve(interpreter, WHEEL_REQUIREMENT) - - return interpreter - - def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): @@ -593,10 +513,7 @@ def to_python_interpreter(full_path_or_basename): pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) - setup_interpreters = [setup_interpreter(interp, - options.interpreter_cache_dir, - options.repos, - options.use_wheel) + setup_interpreters = [vendor.setup_interpreter(interpreter=interp, use_wheel=options.use_wheel) for interp in interpreters] if not setup_interpreters: @@ -637,16 +554,20 @@ def walk_and_do(fn, src_dir): for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) - resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] + resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: - resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) + resolvables.extend(requirements_from_file(requirements_txt, + builder=resolver_option_builder, + interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] - for r in requirements_from_file(constraints_txt, resolver_option_builder): + for r in requirements_from_file(constraints_txt, + builder=resolver_option_builder, + interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) @@ -720,6 +641,8 @@ def main(args=None): if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.') + vendor.adjust_sys_path(include_wheel=options.use_wheel) + if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py index b1f420e10..b0fefc952 100644 --- a/pex/commands/bdist_pex.py +++ b/pex/commands/bdist_pex.py @@ -4,6 +4,7 @@ from setuptools import Command +from pex import vendor from pex.bin.pex import build_pex, configure_clp, make_relative_to_root from pex.common import die from pex.compatibility import ConfigParser, StringIO, string, to_unicode @@ -86,7 +87,8 @@ def run(self): reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): - pex_builder = build_pex(reqs, options, options_builder) + with vendor.adjusted_sys_path(include_wheel=True): + pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() diff --git a/pex/compiler.py b/pex/compiler.py index 3ccd6d036..8a663db56 100644 --- a/pex/compiler.py +++ b/pex/compiler.py @@ -82,7 +82,7 @@ def compile(self, root, relpaths): fp.flush() try: - out, _ = Executor.execute([self._interpreter.binary, fp.name]) + out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( 'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr) diff --git a/pex/environment.py b/pex/environment.py index d40ea1765..7aef1b32d 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -170,12 +170,8 @@ def _resolve(self, working_set, reqs): resolveds.update(working_set.resolve([req], env=self)) except DistributionNotFound as e: TRACER.log('Failed to resolve a requirement: %s' % e) - unresolved_reqs.add(e.args[0].project_name) - # Older versions of pkg_resources just call `DistributionNotFound(req)` instead of the - # modern `DistributionNotFound(req, requirers)` and so we may not have the 2nd requirers - # slot at all. - if len(e.args) >= 2 and e.args[1]: - unresolved_reqs.update(e.args[1]) + unresolved_reqs.add(e.req.project_name) + unresolved_reqs.update(e.requirers_str) unresolved_reqs = set([req.lower() for req in unresolved_reqs]) diff --git a/pex/executor.py b/pex/executor.py index 4541666c4..8f2a42506 100644 --- a/pex/executor.py +++ b/pex/executor.py @@ -56,28 +56,17 @@ def __init__(self, cmd, exc): self.exc = exc @classmethod - def open_process(cls, cmd, env=None, cwd=None, combined=False, **kwargs): + def open_process(cls, cmd, **kwargs): """Opens a process object via subprocess.Popen(). :param string|list cmd: A list or string representing the command to run. - :param dict env: An environment dict for the execution. - :param string cwd: The target cwd for command execution. - :param bool combined: Whether or not to combine stdin and stdout streams. :return: A `subprocess.Popen` object. :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist. """ assert len(cmd) > 0, 'cannot execute an empty command!' try: - return subprocess.Popen( - cmd, - stdin=kwargs.pop('stdin', subprocess.PIPE), - stdout=kwargs.pop('stdout', subprocess.PIPE), - stderr=kwargs.pop('stderr', subprocess.STDOUT if combined else subprocess.PIPE), - cwd=cwd, - env=env, - **kwargs - ) + return subprocess.Popen(cmd, **kwargs) except (IOError, OSError) as e: if e.errno == errno.ENOENT: raise cls.ExecutableNotFound(cmd, e) @@ -85,18 +74,19 @@ def open_process(cls, cmd, env=None, cwd=None, combined=False, **kwargs): raise cls.ExecutionError(repr(e), cmd, e) @classmethod - def execute(cls, cmd, env=None, cwd=None, stdin_payload=None, **kwargs): + def execute(cls, cmd, stdin_payload=None, **kwargs): """Execute a command via subprocess.Popen and returns the stdio. :param string|list cmd: A list or string representing the command to run. - :param dict env: An environment dict for the execution. - :param string cwd: The target cwd for command execution. :param string stdin_payload: A string representing the stdin payload, if any, to send. :return: A tuple of strings representing (stdout, stderr), pre-decoded for utf-8. :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist. `Executor.NonZeroExit` when the execution fails with a non-zero exit code. """ - process = cls.open_process(cmd=cmd, env=env, cwd=cwd, **kwargs) + process = cls.open_process(cmd=cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, **kwargs) stdout_raw, stderr_raw = process.communicate(input=stdin_payload) # N.B. In cases where `stdout` or `stderr` is passed as parameters, these can be None. stdout = stdout_raw.decode('utf-8') if stdout_raw is not None else stdout_raw diff --git a/pex/finders.py b/pex/finders.py index b39f9510f..4fcdcd81f 100644 --- a/pex/finders.py +++ b/pex/finders.py @@ -60,8 +60,6 @@ def __eq__(self, other): # finders together. This is probably possible using importlib but that does us no good as the # importlib machinery supporting this is only available in Python >= 3.1. def _get_finder(importer): - if not hasattr(pkg_resources, '_distribution_finders'): - return None return pkg_resources._distribution_finders.get(importer) @@ -250,22 +248,6 @@ def register_finders(): __PREVIOUS_FINDER = previous_finder -def unregister_finders(): - """Unregister finders necessary for PEX to function properly.""" - - global __PREVIOUS_FINDER - if not __PREVIOUS_FINDER: - return - - pkg_resources.register_finder(zipimport.zipimporter, __PREVIOUS_FINDER) - _remove_finder(pkgutil.ImpImporter, find_wheels_on_path) - - if importlib_machinery is not None: - _remove_finder(importlib_machinery.FileFinder, find_wheels_on_path) - - __PREVIOUS_FINDER = None - - def get_script_from_egg(name, dist): """Returns location, content of script in distribution or (None, None) if not there.""" if dist.metadata_isdir('scripts') and name in dist.metadata_listdir('scripts'): diff --git a/pex/installer.py b/pex/installer.py index 9f933d0e0..ad53faa11 100644 --- a/pex/installer.py +++ b/pex/installer.py @@ -5,9 +5,6 @@ import os import sys -import tempfile - -from pkg_resources import Distribution, PathMetadata from .common import safe_mkdtemp, safe_rmtree from .compatibility import WINDOWS @@ -17,7 +14,6 @@ from .version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT __all__ = ( - 'Installer', 'Packager' ) @@ -26,7 +22,7 @@ def after_installation(function): def function_wrapper(self, *args, **kw): self._installed = self.run() if not self._installed: - raise Installer.InstallFailure('Failed to install %s' % self._source_dir) + raise InstallerBase.InstallFailure('Failed to install %s' % self._source_dir) return function(self, *args, **kw) return function_wrapper @@ -45,19 +41,13 @@ class Error(Exception): pass class InstallFailure(Error): pass class IncapableInterpreter(Error): pass - def __init__(self, source_dir, strict=True, interpreter=None, install_dir=None): - """ - Create an installer from an unpacked source distribution in source_dir. - - If strict=True, fail if any installation dependencies (e.g. distribute) - are missing. - """ + def __init__(self, source_dir, interpreter=None, install_dir=None): + """Create an installer from an unpacked source distribution in source_dir.""" self._source_dir = source_dir self._install_tmp = install_dir or safe_mkdtemp() self._installed = None - self._strict = strict self._interpreter = interpreter or PythonInterpreter.get() - if not self._interpreter.satisfies(self.capability) and strict: + if not self._interpreter.satisfies(self.capability): raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % ( self._interpreter.binary, self.__class__.__name__)) @@ -90,9 +80,6 @@ def bootstrap_script(self): bootstrap_modules = [] for module, requirement in self.mixins().items(): path = self._interpreter.get_location(requirement) - if not path: - assert not self._strict # This should be caught by validation - continue bootstrap_sys_paths.append(self.SETUP_BOOTSTRAP_PYPATH % {'path': path}) bootstrap_modules.append(self.SETUP_BOOTSTRAP_MODULE % {'module': module}) return '\n'.join( @@ -107,7 +94,7 @@ def run(self): return self._installed with TRACER.timed('Installing %s' % self._install_tmp, V=2): - command = [self._interpreter.binary, '-'] + self._setup_command() + command = [self._interpreter.binary, '-sE', '-'] + self._setup_command() try: Executor.execute(command, env=self._interpreter.sanitized_environment(), @@ -128,70 +115,6 @@ def cleanup(self): safe_rmtree(self._install_tmp) -class Installer(InstallerBase): - """Install an unpacked distribution with a setup.py.""" - - def __init__(self, source_dir, strict=True, interpreter=None): - """ - Create an installer from an unpacked source distribution in source_dir. - - If strict=True, fail if any installation dependencies (e.g. setuptools) - are missing. - """ - super(Installer, self).__init__(source_dir, strict=strict, interpreter=interpreter) - self._egg_info = None - fd, self._install_record = tempfile.mkstemp() - os.close(fd) - - def _setup_command(self): - return ['install', - '--root=%s' % self._install_tmp, - '--prefix=', - '--single-version-externally-managed', - '--record', self._install_record] - - def _postprocess(self): - installed_files = [] - egg_info = None - with open(self._install_record) as fp: - installed_files = fp.read().splitlines() - for line in installed_files: - if line.endswith('.egg-info'): - assert line.startswith('/'), 'Expect .egg-info to be within install_tmp!' - egg_info = line - break - - if not egg_info: - self._installed = False - return self._installed - - installed_files = [os.path.relpath(fn, egg_info) for fn in installed_files if fn != egg_info] - - self._egg_info = os.path.join(self._install_tmp, egg_info[1:]) - with open(os.path.join(self._egg_info, 'installed-files.txt'), 'w') as fp: - fp.write('\n'.join(installed_files)) - fp.write('\n') - - return self._installed - - @after_installation - def egg_info(self): - return self._egg_info - - @after_installation - def root(self): - egg_info = self.egg_info() - assert egg_info - return os.path.realpath(os.path.dirname(egg_info)) - - @after_installation - def distribution(self): - base_dir = self.root() - egg_info = self.egg_info() - metadata = PathMetadata(base_dir, egg_info) - return Distribution.from_location(base_dir, os.path.basename(egg_info), metadata=metadata) - - class DistributionPackager(InstallerBase): def mixins(self): mixins = super(DistributionPackager, self).mixins().copy() diff --git a/pex/interpreter.py b/pex/interpreter.py index 0bfa8b22a..1ea844481 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -53,25 +53,8 @@ """ -EXTRAS_PY = b"""\ -try: - import pkg_resources -except ImportError: - sys.exit(0) - -requirements = {} -for item in sys.path: - for dist in pkg_resources.find_distributions(item): - requirements[str(dist.as_requirement())] = dist.location - -for requirement_str, location in requirements.items(): - rs = requirement_str.split('==', 2) - if len(rs) == 2: - print('%s %s %s' % (rs[0], rs[1], location)) -""" - -def _generate_identity_source(include_site_extras): +def _generate_identity_source(): # Determine in the most platform-compatible way possible the identity of the interpreter # and its known packages. encodables = ( @@ -83,11 +66,8 @@ def _generate_identity_source(include_site_extras): get_impl_ver ) - source = ID_PY_TMPL.replace(b'__CODE__', - b'\n\n'.join(getsource(func).encode('utf-8') for func in encodables)) - if include_site_extras: - source += EXTRAS_PY - return source + return ID_PY_TMPL.replace(b'__CODE__', + b'\n\n'.join(getsource(func).encode('utf-8') for func in encodables)) class PythonIdentity(object): @@ -307,17 +287,6 @@ def all(cls, paths=None): paths = os.getenv('PATH', '').split(':') return cls.filter(cls.find(paths)) - @classmethod - def _parse_extras(cls, output_lines): - def iter_lines(): - for line in output_lines: - try: - dist_name, dist_version, location = line.split() - except ValueError: - raise cls.IdentificationError('Could not identify requirement: %s' % line) - yield ((dist_name, dist_version), location) - return dict(iter_lines()) - @staticmethod def _iter_extras(path_extras): for item in path_extras: @@ -326,23 +295,19 @@ def _iter_extras(path_extras): yield ((dist.key, dist.version), dist.location) @classmethod - def _from_binary_internal(cls, path_extras, include_site_extras): - extras = sys.path + list(path_extras) if include_site_extras else list(path_extras) - return cls(sys.executable, PythonIdentity.get(), dict(cls._iter_extras(extras))) + def _from_binary_internal(cls): + return cls(sys.executable, PythonIdentity.get()) @classmethod - def _from_binary_external(cls, binary, path_extras, include_site_extras): + def _from_binary_external(cls, binary): environ = cls.sanitized_environment() - stdout, _ = Executor.execute([binary], + stdout, _ = Executor.execute([binary, '-sE'], env=environ, - stdin_payload=_generate_identity_source(include_site_extras)) - output = stdout.splitlines() - if len(output) == 0: + stdin_payload=_generate_identity_source()) + identity = stdout.strip() + if len(identity) == 0: raise cls.IdentificationError('Could not establish identity of %s' % binary) - identity, raw_extras = output[0], output[1:] - extras = cls._parse_extras(raw_extras) - extras.update(cls._iter_extras(path_extras)) - return cls(binary, PythonIdentity.from_id_string(identity), extras=extras) + return cls(binary, PythonIdentity.from_id_string(identity)) @classmethod def expand_path(cls, path): @@ -369,26 +334,19 @@ def from_env(cls, hashbang): TRACER.log('Could not identify %s: %s' % (fn, e)) @classmethod - def from_binary(cls, binary, path_extras=None, include_site_extras=True): + def from_binary(cls, binary): """Create an interpreter from the given `binary`. :param str binary: The path to the python interpreter binary. - :param path_extras: Extra PYTHONPATH entries to add to the interpreter's `sys.path`. - :type path_extras: list of str - :param bool include_site_extras: `True` to include the `site-packages` associated - with `binary` in the interpreter's `sys.path`. - :return: an interpreter created from the given `binary` with only the specified - extras. + :return: an interpreter created from the given `binary`. :rtype: :class:`PythonInterpreter` """ - path_extras = path_extras or () - key = (binary, tuple(path_extras), include_site_extras) - if key not in cls.CACHE: + if binary not in cls.CACHE: if binary == sys.executable: - cls.CACHE[key] = cls._from_binary_internal(path_extras, include_site_extras) + cls.CACHE[binary] = cls._from_binary_internal() else: - cls.CACHE[key] = cls._from_binary_external(binary, path_extras, include_site_extras) - return cls.CACHE[key] + cls.CACHE[binary] = cls._from_binary_external(binary) + return cls.CACHE[binary] @classmethod def find(cls, paths): @@ -474,7 +432,7 @@ def __init__(self, binary, identity, extras=None): self._identity = identity def with_extra(self, key, version, location): - extras = self._extras.copy() + extras = {(k, v): loc for (k, v), loc in self._extras.items() if k != key} extras[(key, version)] = location return self.__class__(self._binary, self._identity, extras) diff --git a/pex/package.py b/pex/package.py index 3211aa9e4..2723cc089 100644 --- a/pex/package.py +++ b/pex/package.py @@ -78,10 +78,6 @@ def satisfies(self, requirement, allow_prereleases=None): link_name = safe_name(self.name).lower() if link_name != requirement.key: return False - - # NB: If we upgrade to setuptools>=34 the SpecifierSet used here (requirement.specifier) will - # come from a non-vendored `packaging` package and pex's bootstrap code in `PEXBuilder` will - # need an update. return requirement.specifier.contains(self.raw_version, prereleases=allow_prereleases) def compatible(self, supported_tags): diff --git a/pex/pex.py b/pex/pex.py index 83ff127f9..4cb858162 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -517,14 +517,7 @@ def execute_pkg_resources(cls, spec): cls.demote_bootstrap() entry = EntryPoint.parse("run = {0}".format(spec)) - - # See https://pythonhosted.org/setuptools/history.html#id25 for rationale here. - if hasattr(entry, 'resolve'): - # setuptools >= 11.3 - runner = entry.resolve() - else: - # setuptools < 11.3 - runner = entry.load(require=False) + runner = entry.resolve() return runner() def cmdline(self, args=()): @@ -557,9 +550,6 @@ def run(self, args=(), with_chroot=False, blocking=True, setsid=False, **kwargs) process = Executor.open_process(cmdline, cwd=self._pex if with_chroot else os.getcwd(), preexec_fn=os.setsid if setsid else None, - stdin=kwargs.pop('stdin', None), - stdout=kwargs.pop('stdout', None), - stderr=kwargs.pop('stderr', None), **kwargs) return process.wait() if blocking else process @@ -589,5 +579,5 @@ def _do_entry_point_verification(self): retcode = self.run([fp.name], env={'PEX_INTERPRETER': '1'}) if retcode != 0: raise self.InvalidEntryPoint('Invalid entry point: `{}`\n' - 'Entry point verification failed: `{}`' - .format(entry_point, import_statement)) + 'Entry point verification failed: `{}`' + .format(entry_point, import_statement)) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index a12155cf5..073cb83e7 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -393,12 +393,6 @@ def _copy_or_link(self, src, dst, label=None): else: self._chroot.link(src, dst, label) - # TODO(wickman) Ideally we unqualify our setuptools dependency and inherit whatever is - # bundled into the environment so long as it is compatible (and error out if not.) - # - # As it stands, we're picking and choosing the pieces we think we need, which means - # if there are bits of setuptools imported from elsewhere they may be incompatible with - # this. def _prepare_bootstrap(self): # Writes enough of setuptools into the .pex .bootstrap directory so that we can be fully # self-contained. @@ -417,7 +411,7 @@ def _prepare_bootstrap(self): ) for fn, content_stream in DistributionHelper.walk_data(setuptools): - if fn.startswith('pkg_resources') or fn.startswith('_markerlib'): + if fn.startswith('pkg_resources'): if not fn.endswith('.pyc'): # We'll compile our own .pyc's later. dst = os.path.join(self.BOOTSTRAP_DIR, fn) self._chroot.write(content_stream.read(), dst, 'bootstrap') diff --git a/pex/requirements.py b/pex/requirements.py index e53aede81..64e72b32b 100644 --- a/pex/requirements.py +++ b/pex/requirements.py @@ -31,7 +31,7 @@ def __init__(self, filename): # Process lines in the requirements.txt format as defined here: # https://pip.pypa.io/en/latest/reference/pip_install.html#requirements-file-format -def requirements_from_lines(lines, builder=None, relpath=None): +def requirements_from_lines(lines, builder=None, interpreter=None, relpath=None): relpath = relpath or os.getcwd() builder = builder.clone() if builder else ResolverOptionsBuilder() to_resolve = [] @@ -74,17 +74,21 @@ def requirements_from_lines(lines, builder=None, relpath=None): for resolvable in to_resolve: if isinstance(resolvable, RequirementsTxtSentinel): - resolvables.extend(requirements_from_file(resolvable.filename, builder=builder)) + resolvables.extend(requirements_from_file(resolvable.filename, + builder=builder, + interpreter=interpreter)) else: try: - resolvables.append(Resolvable.get(resolvable, builder)) + resolvables.append(Resolvable.get(resolvable, + options_builder=builder, + interpreter=interpreter)) except Resolvable.Error as e: raise UnsupportedLine('Could not resolve line: %s (%s)' % (resolvable, e)) return resolvables -def requirements_from_file(filename, builder=None): +def requirements_from_file(filename, builder=None, interpreter=None): """Return a list of :class:`Resolvable` objects from a requirements.txt file. :param filename: The filename of the requirements.txt @@ -95,4 +99,7 @@ def requirements_from_file(filename, builder=None): relpath = os.path.dirname(filename) with open(filename, 'r') as fp: - return requirements_from_lines(fp.readlines(), builder=builder, relpath=relpath) + return requirements_from_lines(fp.readlines(), + builder=builder, + interpreter=interpreter, + relpath=relpath) diff --git a/pex/resolvable.py b/pex/resolvable.py index f82a040ce..518f7d886 100644 --- a/pex/resolvable.py +++ b/pex/resolvable.py @@ -57,7 +57,7 @@ def register(cls, implementation): cls._REGISTRY.append(implementation) @classmethod - def get(cls, resolvable_string, options_builder=None): + def get(cls, resolvable_string, options_builder=None, interpreter=None): """Get a :class:`Resolvable` from a string. :returns: A :class:`Resolvable` or ``None`` if no implementation was appropriate. @@ -65,14 +65,16 @@ def get(cls, resolvable_string, options_builder=None): options_builder = options_builder or ResolverOptionsBuilder() for resolvable_impl in cls._REGISTRY: try: - return resolvable_impl.from_string(resolvable_string, options_builder) + return resolvable_impl.from_string(resolvable_string, + options_builder, + interpreter=interpreter) except cls.InvalidRequirement: continue raise cls.InvalidRequirement('Unknown requirement type: %s' % resolvable_string) # @abstractmethod - Only available in Python 3.3+ @classmethod - def from_string(cls, requirement_string, options_builder): + def from_string(cls, requirement_string, options_builder, interpreter=None): """Produce a resolvable from this requirement string. :returns: Instance of the particular Resolvable implementation. @@ -117,10 +119,8 @@ def name(self): def exact(self): """Whether or not this resolvable specifies an exact (cacheable) requirement.""" - # TODO(wickman) Currently 'interpreter' is unused but it is reserved for environment - # marker evaluation per PEP426 and: - # https://bitbucket.org/pypa/setuptools/issue/353/allow-distributionrequires-be-evaluated - def extras(self, interpreter=None): + @property + def extras(self): """Return the "extras" tags associated with this resolvable if any.""" return [] @@ -131,7 +131,7 @@ class ResolvableRepository(Resolvable): COMPATIBLE_VCS = frozenset(['git', 'svn', 'hg', 'bzr']) @classmethod - def from_string(cls, requirement_string, options_builder): + def from_string(cls, requirement_string, options_builder, interpreter=None): if any(requirement_string.startswith('%s+' % vcs) for vcs in cls.COMPATIBLE_VCS): # further delegate pass @@ -162,7 +162,7 @@ class ResolvablePackage(Resolvable): # TODO(wickman) Implement extras parsing for ResolvablePackage @classmethod - def from_string(cls, requirement_string, options_builder): + def from_string(cls, requirement_string, options_builder, interpreter=None): requirement_string, extras = strip_extras(requirement_string) package = Package.from_href(requirement_string) if package is None: @@ -188,7 +188,8 @@ def name(self): def exact(self): return True - def extras(self, interpreter=None): + @property + def extras(self): return self._extras def __eq__(self, other): @@ -205,7 +206,7 @@ class ResolvableRequirement(Resolvable): """A requirement (e.g. 'setuptools', 'Flask>=0.8,<0.9', 'pex[whl]').""" @classmethod - def from_string(cls, requirement_string, options_builder): + def from_string(cls, requirement_string, options_builder, interpreter=None): try: req = maybe_requirement(requirement_string) except ValueError: @@ -234,7 +235,8 @@ def name(self): def exact(self): return requirement_is_exact(self.requirement) - def extras(self, interpreter=None): + @property + def extras(self): return list(self.requirement.extras) def __eq__(self, other): @@ -251,28 +253,30 @@ class ResolvableDirectory(ResolvablePackage): """A source directory (with setup.py) resolvable.""" @classmethod - def is_installable(cls, requirement_string): + def _is_installable(cls, requirement_string): if not os.path.isdir(requirement_string): return False return os.path.isfile(os.path.join(requirement_string, 'setup.py')) @classmethod - def from_string(cls, requirement_string, options_builder): + def from_string(cls, requirement_string, options_builder, interpreter=None): requirement_string, extras = strip_extras(requirement_string) - if cls.is_installable(requirement_string): + if cls._is_installable(requirement_string): + if interpreter is None: + raise cls.InvalidRequirement('%s is not an installable directory because we were called ' + 'without an interpreter to use to execute %s.' + % (requirement_string, 'setup.py')) + packager = Packager(requirement_string, interpreter=interpreter) try: - # TODO(wickman) This is one case where interpreter is necessary to be fully correct. This - # may indicate that packages() should take interpreter like extras does. Once we have - # metadata in setup.cfg or whatever, then we can get the interpreter out of the equation. - sdist = Packager(requirement_string).sdist() + sdist = packager.sdist() except InstallerBase.Error: raise cls.InvalidRequirement('Could not create source distribution for %s' % - requirement_string) + requirement_string) package = Package.from_href(sdist) return ResolvablePackage(package, options_builder.build(package.name), extras=extras) else: raise cls.InvalidRequirement('%s does not appear to be an installable directory.' - % requirement_string) + % requirement_string) Resolvable.register(ResolvableDirectory) @@ -283,7 +287,7 @@ def from_string(cls, requirement_string, options_builder): # TODO(wickman) Because we explicitly acknowledge all implementations of Resolvable here, # perhaps move away from a registry pattern and integrate into Resolvable classmethod. -def resolvables_from_iterable(iterable, builder): +def resolvables_from_iterable(iterable, builder, interpreter=None): """Given an iterable of resolvable-like objects, return list of Resolvable objects. :param iterable: An iterable of :class:`Resolvable`, :class:`Requirement`, :class:`Package`, @@ -299,7 +303,7 @@ def translate(obj): elif isinstance(obj, Package): return ResolvablePackage(obj, builder.build(obj.name)) elif isinstance(obj, compatibility_string): - return Resolvable.get(obj, builder) + return Resolvable.get(obj, builder, interpreter=interpreter) else: raise ValueError('Do not know how to resolve %s' % type(obj)) return list(map(translate, iterable)) diff --git a/pex/resolver.py b/pex/resolver.py index 4f19d5f4a..2d4ed894a 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -142,7 +142,7 @@ def packages(self): def extras(self, name): return set.union( - *[set(tup.resolvable.extras()) for tup in self.__tuples + *[set(tup.resolvable.extras) for tup in self.__tuples if self.normalize(tup.resolvable.name) == self.normalize(name)]) def replace_built(self, built_packages): @@ -499,7 +499,7 @@ def resolve(requirements, interpreter=interpreter, platform=platform) - return resolver.resolve(resolvables_from_iterable(requirements, builder)) + return resolver.resolve(resolvables_from_iterable(requirements, builder, interpreter=interpreter)) def resolve_multi(requirements, diff --git a/pex/testing.py b/pex/testing.py index d298168d5..fecbe0665 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -12,14 +12,14 @@ from collections import namedtuple from textwrap import dedent +from . import vendor from .bin.pex import log, main from .common import open_zip, safe_mkdir, safe_rmtree, touch from .compatibility import PY3, nested -from .executor import Executor from .installer import EggInstaller, Packager +from .pex import PEX from .pex_builder import PEXBuilder from .util import DistributionHelper, named_temporary_file -from .version import SETUPTOOLS_REQUIREMENT IS_PYPY = "hasattr(sys, 'pypy_version_info')" NOT_CPYTHON27 = ("%s or (sys.version_info[0], sys.version_info[1]) != (2, 7)" % (IS_PYPY)) @@ -127,14 +127,22 @@ def write_zipfile(directory, dest, reverse=False): @contextlib.contextmanager -def make_installer(name='my_project', version='0.0.0', installer_impl=EggInstaller, zip_safe=True, - install_reqs=None, **kwargs): +def make_installer(name='my_project', + version='0.0.0', + installer_impl=EggInstaller, + zip_safe=True, + install_reqs=None, + interpreter=None, + **kwargs): interp = {'project_name': name, 'version': version, 'zip_safe': zip_safe, 'install_requires': install_reqs or []} with temporary_content(PROJECT_CONTENT, interp=interp) as td: - yield installer_impl(td, **kwargs) + yield installer_impl(td, + interpreter=vendor.setup_interpreter(interpreter=interpreter, + use_wheel=True), + **kwargs) @contextlib.contextmanager @@ -201,22 +209,25 @@ def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) - pb = PEXBuilder(path=td, - preamble=COVERAGE_PREAMBLE if coverage else None, - interpreter=interpreter) + # PEXBuilder uses pkg_resources and also wheel if wheel dists are added. Since we don't know here + # if a test will hand us wheels or not, assume they might and setup the PEXBuilder to handle them. + with vendor.adjusted_sys_path(include_wheel=True): + pb = PEXBuilder(path=td, + preamble=COVERAGE_PREAMBLE if coverage else None, + interpreter=vendor.setup_interpreter(interpreter=interpreter, use_wheel=True)) - for dist in dists: - pb.add_dist_location(dist.location) + for dist in dists: + pb.add_dist_location(dist.location) - for env_filename, contents in sources: - src_path = os.path.join(td, env_filename) - safe_mkdir(os.path.dirname(src_path)) - with open(src_path, 'w') as fp: - fp.write(contents) - pb.add_source(src_path, env_filename) + for env_filename, contents in sources: + src_path = os.path.join(td, env_filename) + safe_mkdir(os.path.dirname(src_path)) + with open(src_path, 'w') as fp: + fp.write(contents) + pb.add_source(src_path, env_filename) - pb.set_executable(os.path.join(td, 'exe.py')) - pb.freeze() + pb.set_executable(os.path.join(td, 'exe.py')) + pb.freeze() return pb @@ -236,7 +247,7 @@ def assert_failure(self): assert self.exception or self.return_code -def run_pex_command(args, env=None): +def run_pex_command(args): """Simulate running pex command for integration testing. This is different from run_simple_pex in that it calls the pex command rather @@ -267,46 +278,26 @@ def mock_logger(msg, v=None): return IntegResults(output, error_code, exception, tb) -# TODO(wickman) Why not PEX.run? -def run_simple_pex(pex, args=(), env=None, stdin=None): - process = Executor.open_process([sys.executable, pex] + list(args), env=env, combined=True) +def run_simple_pex(pex, args=(), interpreter=None, stdin=None, **kwargs): + p = PEX(pex, interpreter=vendor.setup_interpreter(interpreter, use_wheel=True)) + process = p.run(args=args, + blocking=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + **kwargs) stdout, _ = process.communicate(input=stdin) print(stdout.decode('utf-8') if PY3 else stdout) return stdout.replace(b'\r', b''), process.returncode -def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False): +def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False, interpreter=None): with nested(temporary_dir(), temporary_dir()) as (td1, td2): - pb = write_simple_pex(td1, body, dists=dists, coverage=coverage) + interpreter = vendor.setup_interpreter(interpreter, use_wheel=True) + pb = write_simple_pex(td1, body, dists=dists, coverage=coverage, interpreter=interpreter) pex = os.path.join(td2, 'app.pex') pb.build(pex) - return run_simple_pex(pex, args=args, env=env) - - -def _iter_filter(data_dict): - fragment = '/%s/_pex/' % PEXBuilder.BOOTSTRAP_DIR - for filename, records in data_dict.items(): - try: - bi = filename.index(fragment) - except ValueError: - continue - # rewrite to look like root source - yield ('pex/' + filename[bi + len():], records) - - -def combine_pex_coverage(coverage_file_iter): - from coverage.data import CoverageData - - combined = CoverageData(basename='.coverage_combined') - - for filename in coverage_file_iter: - cov = CoverageData(basename=filename) - cov.read() - combined.add_line_data(dict(_iter_filter(cov.line_data()))) - combined.add_arc_data(dict(_iter_filter(cov.arc_data()))) - - combined.write() - return combined.filename + return run_simple_pex(pex, args=args, env=env, interpreter=interpreter) def bootstrap_python_installer(dest): @@ -362,7 +353,6 @@ def ensure_python_distribution(version): env['CONFIGURE_OPTS'] = '--enable-shared' subprocess.check_call([pyenv, 'install', '--keep', version], env=env) subprocess.check_call([pip, 'install', '-U', 'pip']) - subprocess.check_call([pip, 'install', SETUPTOOLS_REQUIREMENT]) python = os.path.join(interpreter_location, 'bin', 'python' + version[0:3]) return python, pip diff --git a/pex/translator.py b/pex/translator.py index b8fc98d13..b9d925f7a 100644 --- a/pex/translator.py +++ b/pex/translator.py @@ -95,10 +95,7 @@ def translate(self, package, into=None): if self._use_2to3 and version >= (3,): with TRACER.timed('Translating 2->3 %s' % package.name): self.run_2to3(unpack_path) - installer = self._installer_impl( - unpack_path, - interpreter=self._interpreter, - strict=(package.name not in ('distribute', 'setuptools'))) + installer = self._installer_impl(unpack_path, interpreter=self._interpreter) with TRACER.timed('Packaging %s' % package.name): try: dist_path = installer.bdist() diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py new file mode 100644 index 000000000..c3bae3c7e --- /dev/null +++ b/pex/vendor/__init__.py @@ -0,0 +1,72 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import collections +import importlib +import os +import sys + +from contextlib import contextmanager + +from pex.interpreter import PythonInterpreter +from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT + + +class VendorSpec(collections.namedtuple('VendorSpec', ['key', 'version', 'target_dir'])): + VENDOR_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '_vendored')) + + @classmethod + def create(cls, requirement): + components = requirement.rsplit('==', 1) + if len(components) != 2: + raise ValueError('Vendored requirements must be pinned, given {!r}'.format(requirement)) + key, version = tuple(c.strip() for c in components) + return cls(key=key, version=version, target_dir=os.path.join(cls.VENDOR_DIR, key)) + + @property + def requirement(self): + return '{}=={}'.format(self.key, self.version) + + +__SETUPTOOLS = VendorSpec.create(SETUPTOOLS_REQUIREMENT) +__WHEEL = VendorSpec.create(WHEEL_REQUIREMENT) + + +def vendor_specs(): + return __SETUPTOOLS, __WHEEL + + +def adjust_sys_path(include_wheel=False): + vendored_path = [__SETUPTOOLS.target_dir] + if include_wheel: + vendored_path.append(__WHEEL.target_dir) + + num_entries = len(vendored_path) + inserted = 0 + if sys.path[:num_entries] != vendored_path: + for path in reversed(vendored_path): + sys.path.insert(0, path) + inserted = len(vendored_path) + return inserted, vendored_path + + +@contextmanager +def adjusted_sys_path(include_wheel=False): + inserted_count, vendored_path = adjust_sys_path(include_wheel=include_wheel) + try: + yield vendored_path[:] + finally: + del sys.path[:inserted_count] + + +def _resolve_vendored(include_wheel=False): + with adjusted_sys_path(include_wheel=include_wheel) as vendored_path: + pkg_resources = importlib.import_module('pkg_resources') + return list(pkg_resources.WorkingSet(entries=vendored_path)) + + +def setup_interpreter(interpreter=None, use_wheel=False): + interpreter = interpreter or PythonInterpreter.get() + for dist in _resolve_vendored(include_wheel=use_wheel): + interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) + return interpreter diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py new file mode 100644 index 000000000..b53ba464f --- /dev/null +++ b/pex/vendor/__main__.py @@ -0,0 +1,43 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import print_function + +import os +import subprocess +import sys + +from pex.common import safe_delete, safe_rmtree + +from . import vendor_specs + + +class VendorizeError(Exception): + """Indicates an error was encountered updating vendored libraries.""" + + +def vendorize(vendor_spec): + cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir, + vendor_spec.requirement] + result = subprocess.call(cmd) + if result != 0: + raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec)) + + # We know we can get these as a by-product of a pip install but never need them. + safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin')) + safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py')) + + +if __name__ == '__main__': + if len(sys.argv) != 1: + print('Usage: {}'.format(sys.argv[0]), file=sys.stderr) + sys.exit(1) + + try: + for vendor_spec in vendor_specs(): + vendorize(vendor_spec) + print('Vendored {!r}.'.format(vendor_spec)) + sys.exit(0) + except VendorizeError as e: + print('Problem encountered vendorizing: {}'.format(e), file=sys.stderr) + sys.exit(1) diff --git a/pex/version.py b/pex/version.py index 5a61411b2..566577a30 100644 --- a/pex/version.py +++ b/pex/version.py @@ -3,8 +3,8 @@ __version__ = '1.5.1' -# Versions 34.0.0 through 35.0.2 (last pre-36.0.0) de-vendored dependencies which causes problems -# for pex code so we exclude that range. -SETUPTOOLS_REQUIREMENT = 'setuptools>=20.3,<41,!=34.*,!=35.*' +SETUPTOOLS_REQUIREMENT = 'setuptools==40.4.3' -WHEEL_REQUIREMENT = 'wheel>=0.26.0,<0.32' +# We're currently stuck here due to removal of an API we depend on. +# See: https://github.com/pantsbuild/pex/issues/603 +WHEEL_REQUIREMENT = 'wheel==0.31.1' diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 25ed07a73..bac8bc3d8 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash coverage run -p -m py.test tests coverage run -p -m pex.bin.pex -v --help >&/dev/null diff --git a/scripts/style.sh b/scripts/style.sh new file mode 100755 index 000000000..eda83d36e --- /dev/null +++ b/scripts/style.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +ROOT_DIR="$(git rev-parse --show-toplevel)" + +twitterstyle -n ImportOrder "${ROOT_DIR}/tests" $( + find "${ROOT_DIR}/pex" -path "${ROOT_DIR}/pex/vendor/_vendored" -prune , -name "*.py" +) \ No newline at end of file diff --git a/setup.py b/setup.py index 3b7f3b404..e3fcfc442 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,27 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys -from setuptools import setup -with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as fp: +# We may be executed from outside the project dir: `python pex/setup.py ...`, so ensure the +# `setup.py` dir is on the path. +__HERE = os.path.realpath(os.path.dirname(__file__)) + + +sys.path.append(__HERE) +from pex import vendor +vendor.adjust_sys_path(include_wheel=True) + + +os.chdir(__HERE) +from setuptools import find_packages, setup + + +with open(os.path.join(__HERE, 'README.rst')) as fp: LONG_DESCRIPTION = fp.read() + '\n' -with open(os.path.join(os.path.dirname(__file__), 'CHANGES.rst')) as fp: +with open(os.path.join(__HERE, 'CHANGES.rst')) as fp: LONG_DESCRIPTION += fp.read() @@ -16,24 +30,24 @@ # # Populates the following variables: # __version__ -# __setuptools_requirement -# __wheel_requirement +# SETUPTOOLS_REQUIREMENT +# WHEEL_REQUIREMENT __version__ = '' -version_py_file = os.path.join(os.path.dirname(__file__), 'pex', 'version.py') +version_py_file = os.path.join(__HERE, 'pex', 'version.py') with open(version_py_file) as version_py: exec(compile(version_py.read(), version_py_file, 'exec')) setup( - name = 'pex', - version = __version__, - description = "The PEX packaging toolchain.", - long_description = LONG_DESCRIPTION, + name='pex', + version=__version__, + description="The PEX packaging toolchain.", + long_description=LONG_DESCRIPTION, long_description_content_type="text/x-rst", - url = 'https://github.com/pantsbuild/pex', - license = 'Apache License, Version 2.0', - zip_safe = True, - classifiers = [ + url='https://github.com/pantsbuild/pex', + license='Apache License, Version 2.0', + zip_safe=False, + classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: Unix', @@ -48,15 +62,8 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', ], - packages = [ - 'pex', - 'pex.bin', - 'pex.commands', - ], - install_requires = [ - SETUPTOOLS_REQUIREMENT, - WHEEL_REQUIREMENT, - ], + packages=find_packages(), + include_package_data=True, extras_require={ # For improved subprocess robustness under python2.7. 'subprocess': ['subprocess32>=3.2.7'], @@ -65,15 +72,7 @@ # For improved requirement resolution and fetching performance. 'cachecontrol': ['CacheControl>=0.12.3'], }, - tests_require = [ - 'mock', - 'twitter.common.contextutil>=0.3.1,<0.4.0', - 'twitter.common.lang>=0.3.1,<0.4.0', - 'twitter.common.testing>=0.3.1,<0.4.0', - 'twitter.common.dirutil>=0.3.1,<0.4.0', - 'pytest', - ], - entry_points = { + entry_points={ 'distutils.commands': [ 'bdist_pex = pex.commands.bdist_pex:bdist_pex', ], diff --git a/tests/test_environment.py b/tests/test_environment.py index 4cdefa3bc..3e72bbd8c 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -6,23 +6,21 @@ from contextlib import contextmanager import pytest -from twitter.common.contextutil import temporary_dir -from pex import resolver +from pex import resolver, vendor from pex.compatibility import nested, to_bytes from pex.environment import PEXEnvironment from pex.installer import EggInstaller, WheelInstaller from pex.interpreter import PythonInterpreter -from pex.package import EggPackage, SourcePackage, WheelPackage from pex.pex import PEX from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pex.testing import make_bdist, temporary_filename -from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT +from pex.testing import make_bdist, temporary_dir, temporary_filename @contextmanager def yield_pex_builder(zip_safe=True, installer_impl=EggInstaller, interpreter=None): + interpreter = vendor.setup_interpreter(interpreter=interpreter, use_wheel=True) with nested(temporary_dir(), make_bdist('p1', zipped=True, @@ -118,29 +116,18 @@ def test_load_internal_cache_unzipped(): reason='Test requires known bad Apple interpreter {}' .format(_KNOWN_BAD_APPLE_INTERPRETER)) def test_osx_platform_intel_issue_523(): - def bad_interpreter(include_site_extras=True): - return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER, - include_site_extras=include_site_extras) + def bad_interpreter(): + return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER) - interpreter = bad_interpreter(include_site_extras=False) with temporary_dir() as cache: # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to - # successfully install psutil. - for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): - for resolved_dist in resolver.resolve([requirement], - cache=cache, - # We can't use wheels since we're bootstrapping them. - precedence=(SourcePackage, EggPackage), - interpreter=interpreter): - dist = resolved_dist.distribution - interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) - - with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), + # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored + # setuptools and wheel extras. + with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()), temporary_filename()) as (pb, pex_file): for resolved_dist in resolver.resolve(['psutil==5.4.3'], cache=cache, - precedence=(SourcePackage, WheelPackage), - interpreter=interpreter): + interpreter=pb.interpreter): pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) diff --git a/tests/test_executor.py b/tests/test_executor.py index ed688e9e5..c3940a0f0 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -2,12 +2,13 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import subprocess import pytest -from twitter.common.contextutil import temporary_dir from pex.common import safe_mkdir from pex.executor import Executor +from pex.testing import temporary_dir TEST_EXECUTABLE = '/a/nonexistent/path/to/nowhere' TEST_CMD_LIST = [TEST_EXECUTABLE, '--version'] @@ -25,7 +26,9 @@ def test_executor_open_process_wait_return(): def test_executor_open_process_communicate(): - process = Executor.open_process(['/bin/echo', '-n', 'hello']) + process = Executor.open_process(['/bin/echo', '-n', 'hello'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert stdout.decode('utf-8') == 'hello' assert stderr.decode('utf-8') == '' @@ -34,6 +37,7 @@ def test_executor_open_process_communicate(): def test_executor_execute(): assert Executor.execute('/bin/echo -n stdout >&1', shell=True) == ('stdout', '') assert Executor.execute('/bin/echo -n stderr >&2', shell=True) == ('', 'stderr') + assert Executor.execute('/bin/echo -n TEST | tee /dev/stderr', shell=True) == ('TEST', 'TEST') assert Executor.execute(['/bin/echo', 'hello']) == ('hello\n', '') assert Executor.execute(['/bin/echo', '-n', 'hello']) == ('hello', '') assert Executor.execute('/bin/echo -n $HELLO', env={'HELLO': 'hey'}, shell=True) == ('hey', '') @@ -43,20 +47,6 @@ def test_executor_execute_zero(): Executor.execute('exit 0', shell=True) -def test_executor_execute_stdio(): - with temporary_dir() as tmp: - with open(os.path.join(tmp, 'stdout'), 'w+b') as fake_stdout: - with open(os.path.join(tmp, 'stderr'), 'w+b') as fake_stderr: - Executor.execute('/bin/echo -n TEST | tee /dev/stderr', - shell=True, - stdout=fake_stdout, - stderr=fake_stderr) - fake_stdout.seek(0) - fake_stderr.seek(0) - assert fake_stdout.read().decode('utf-8') == 'TEST' - assert fake_stderr.read().decode('utf-8') == 'TEST' - - @pytest.mark.parametrize('testable', [Executor.open_process, Executor.execute]) def test_executor_execute_not_found(testable): with pytest.raises(Executor.ExecutableNotFound) as exc: diff --git a/tests/test_inherits_path_option.py b/tests/test_inherits_path_option.py index 3a5f7a8d1..63596b551 100644 --- a/tests/test_inherits_path_option.py +++ b/tests/test_inherits_path_option.py @@ -4,10 +4,9 @@ import os from contextlib import contextmanager -from twitter.common.contextutil import environment_as, temporary_dir - +from pex import vendor from pex.pex_builder import PEXBuilder -from pex.testing import run_simple_pex +from pex.testing import run_simple_pex, temporary_dir @contextmanager @@ -22,13 +21,13 @@ def write_and_run_simple_pex(inheriting=False): with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write('') # No contents, we just want the startup messages - pb = PEXBuilder(path=td, preamble=None) - pb.info.inherit_path = inheriting - pb.set_executable(os.path.join(td, 'exe.py')) - pb.freeze() - pb.build(pex_path) - with environment_as(PEX_VERBOSE='1'): - yield run_simple_pex(pex_path)[0] + # PEXBuilder uses pkg_resources. + with vendor.adjusted_sys_path(): + pb = PEXBuilder(path=td, preamble=None, interpreter=vendor.setup_interpreter()) + pb.info.inherit_path = inheriting + pb.set_executable(os.path.join(td, 'exe.py')) + pb.build(pex_path) + yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0] def test_inherits_path_fallback_option(): diff --git a/tests/test_installer.py b/tests/test_installer.py index 95340f62b..04bbbf581 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -5,40 +5,34 @@ import pytest -from pex.bin.pex import setup_interpreter from pex.installer import WheelInstaller from pex.interpreter import PythonInterpreter -from pex.testing import PY36, ensure_python_interpreter, make_installer, temporary_dir +from pex.testing import PY36, ensure_python_interpreter, make_installer +from pex.vendor import setup_interpreter from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT class OrderableInstaller(WheelInstaller): - def __init__(self, source_dir, strict=True, interpreter=None, install_dir=None, mixins=None): + def __init__(self, source_dir, interpreter=None, install_dir=None, mixins=None): self._mixins = mixins - super(OrderableInstaller, self).__init__(source_dir, strict, interpreter, install_dir) + super(OrderableInstaller, self).__init__(source_dir, interpreter, install_dir) def mixins(self): return self._mixins -@contextlib.contextmanager -def bare_interpreter(): - with temporary_dir() as interpreter_cache: - yield setup_interpreter( - interpreter=PythonInterpreter.from_binary(ensure_python_interpreter(PY36)), - interpreter_cache_dir=interpreter_cache, - repos=None, - use_wheel=True - ) +def bare_interpreter(use_wheel): + interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(PY36)) + return setup_interpreter(interpreter=interpreter, use_wheel=use_wheel) @contextlib.contextmanager def wheel_installer(*mixins): - with bare_interpreter() as interpreter: - with make_installer(installer_impl=OrderableInstaller, - interpreter=interpreter, - mixins=OrderedDict(mixins)) as installer: - yield installer + interpreter = bare_interpreter(use_wheel=WHEEL_EXTRA in mixins) + with make_installer(installer_impl=OrderableInstaller, + interpreter=interpreter, + mixins=OrderedDict(mixins)) as installer: + yield installer WHEEL_EXTRA = ('wheel', WHEEL_REQUIREMENT) diff --git a/tests/test_integration.py b/tests/test_integration.py index 9e53b5fb4..fe3f1f88c 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,9 +10,10 @@ from textwrap import dedent import pytest -from twitter.common.contextutil import environment_as, temporary_dir +from twitter.common.contextutil import environment_as -from pex.compatibility import WINDOWS, to_bytes +from pex import vendor +from pex.compatibility import WINDOWS, nested, to_bytes from pex.installer import EggInstaller from pex.pex_bootstrapper import get_pex_info from pex.testing import ( @@ -29,7 +30,8 @@ run_pex_command, run_simple_pex, run_simple_pex_test, - temporary_content + temporary_content, + temporary_dir ) from pex.util import DistributionHelper, named_temporary_file @@ -52,40 +54,32 @@ def test_pex_raise(): def test_pex_root(): - with temporary_dir() as tmp_home: - with environment_as(HOME=tmp_home): - with temporary_dir() as td: - with temporary_dir() as output_dir: - env = make_env(PEX_INTERPRETER=1) - - output_path = os.path.join(output_dir, 'pex.pex') - args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)] - results = run_pex_command(args=args, env=env) - results.assert_success() - assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' - assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' - assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.' + with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td, output_dir, tmp_home): + with environment_as(HOME=tmp_home, PEX_INTERPRETER='1'): + output_path = os.path.join(output_dir, 'pex.pex') + args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)] + results = run_pex_command(args=args) + results.assert_success() + assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' + assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' + assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.' def test_cache_disable(): - with temporary_dir() as tmp_home: - with environment_as(HOME=tmp_home): - with temporary_dir() as td: - with temporary_dir() as output_dir: - env = make_env(PEX_INTERPRETER=1) - - output_path = os.path.join(output_dir, 'pex.pex') - args = [ - 'pex', - '-o', output_path, - '--not-zip-safe', - '--disable-cache', - '--pex-root={0}'.format(td), - ] - results = run_pex_command(args=args, env=env) - results.assert_success() - assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' - assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' + with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td, output_dir, tmp_home): + with environment_as(HOME=tmp_home, PEX_INTERPRETER='1'): + output_path = os.path.join(output_dir, 'pex.pex') + args = [ + 'pex', + '-o', output_path, + '--not-zip-safe', + '--disable-cache', + '--pex-root={0}'.format(td), + ] + results = run_pex_command(args=args) + results.assert_success() + assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' + assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' def test_pex_interpreter(): @@ -108,7 +102,6 @@ def test_pex_repl_cli(): # Create a temporary pex containing just `requests` with no entrypoint. pex_path = os.path.join(output_dir, 'pex.pex') results = run_pex_command(['--disable-cache', - 'wheel', 'requests', './', '-e', 'pex.bin.pex:main', @@ -173,7 +166,7 @@ def do_something(): """ % error_msg) with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir: - installer = EggInstaller(project_dir) + installer = EggInstaller(project_dir, interpreter=vendor.setup_interpreter()) dist = DistributionHelper.distribution_from_path(installer.bdist()) so, rc = run_simple_pex_test('', env=make_env(PEX_SCRIPT='my_app'), dists=[dist]) assert so.decode('utf-8').strip() == error_msg @@ -453,10 +446,10 @@ def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): '-o', pex_out_path]) res.assert_success() - stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdin_payload = b'import os, sys; print(os.path.realpath(sys.executable)); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 - assert str(sys.executable).encode() in stdout + assert os.path.realpath(sys.executable).encode() in stdout @pytest.mark.skipif(IS_PYPY) diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index 4a6126f46..ff31540f6 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -2,20 +2,12 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import subprocess import pytest from pex import interpreter from pex.compatibility import PY3 -from pex.testing import ( - IS_PYPY, - PY27, - PY35, - ensure_python_distribution, - ensure_python_interpreter, - temporary_dir -) +from pex.testing import IS_PYPY, PY27, PY35, ensure_python_interpreter try: from mock import patch @@ -58,7 +50,7 @@ def test_interpreter_versioning(self, test_interpreter1): assert py_interpreter.identity.version == self.TEST_INTERPRETER1_VERSION_TUPLE @pytest.mark.skipif(IS_PYPY) - def test_interpreter_caching_basic(self, test_interpreter1, test_interpreter2): + def test_interpreter_caching(self, test_interpreter1, test_interpreter2): py_interpreter1 = interpreter.PythonInterpreter.from_binary(test_interpreter1) py_interpreter2 = interpreter.PythonInterpreter.from_binary(test_interpreter2) assert py_interpreter1 is not py_interpreter2 @@ -66,36 +58,3 @@ def test_interpreter_caching_basic(self, test_interpreter1, test_interpreter2): py_interpreter3 = interpreter.PythonInterpreter.from_binary(test_interpreter1) assert py_interpreter1 is py_interpreter3 - - @pytest.mark.skipif(IS_PYPY) - def test_interpreter_caching_include_site_extras(self, test_interpreter1): - py_interpreter1 = interpreter.PythonInterpreter.from_binary(test_interpreter1, - include_site_extras=False) - py_interpreter2 = interpreter.PythonInterpreter.from_binary(test_interpreter1, - include_site_extras=True) - py_interpreter3 = interpreter.PythonInterpreter.from_binary(test_interpreter1) - assert py_interpreter1 is not py_interpreter2 - assert py_interpreter1.identity.version == py_interpreter2.identity.version - assert py_interpreter2 is py_interpreter3 - - @pytest.mark.skipif(IS_PYPY) - def test_interpreter_caching_path_extras(self): - python, pip = ensure_python_distribution(self.TEST_INTERPRETER1_VERSION) - with temporary_dir() as td: - path_extra = os.path.realpath(td) - subprocess.check_call([pip, - 'install', - '--target={}'.format(path_extra), - 'ansicolors==1.1.8']) - py_interpreter1 = interpreter.PythonInterpreter.from_binary(python, - path_extras=[path_extra], - include_site_extras=False) - py_interpreter2 = interpreter.PythonInterpreter.from_binary(python, - include_site_extras=False) - py_interpreter3 = interpreter.PythonInterpreter.from_binary(python, - path_extras=[path_extra], - include_site_extras=False) - assert py_interpreter1 is not py_interpreter2 - assert py_interpreter1.extras == {('ansicolors', '1.1.8'): path_extra} - assert py_interpreter2.extras == {} - assert py_interpreter1 is py_interpreter3 diff --git a/tests/test_pex.py b/tests/test_pex.py index d03a3f95c..c9261f70b 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -9,9 +9,8 @@ from types import ModuleType import pytest -from twitter.common.contextutil import temporary_file -from pex.bin.pex import setup_interpreter +from pex import vendor from pex.compatibility import PY2, WINDOWS, nested, to_bytes from pex.installer import EggInstaller, WheelInstaller from pex.interpreter import PythonInterpreter @@ -271,20 +270,19 @@ def test_pex_paths(): @contextmanager def _add_test_hello_to_pex(ep): - with temporary_dir() as td: - hello_file = "\n".join([ - "def hello():", - " print('hello')", - ]) - with temporary_file(root_dir=td) as tf: - with open(tf.name, 'w') as handle: - handle.write(hello_file) - - pex_builder = PEXBuilder() + hello_file = textwrap.dedent(""" + def hello(): + print('hello') + """) + with named_temporary_file() as tf: + tf.write(to_bytes(hello_file)) + tf.close() + with vendor.adjusted_sys_path(): + pex_builder = PEXBuilder(interpreter=vendor.setup_interpreter()) pex_builder.add_source(tf.name, 'test.py') pex_builder.set_entry_point(ep) pex_builder.freeze() - yield pex_builder + yield pex_builder def test_pex_verify_entry_point_method_should_pass(): @@ -325,10 +323,8 @@ def test_pex_verify_entry_point_module_should_fail(): def test_activate_interpreter_different_from_current(): with temporary_dir() as pex_root: interp_version = PY36 if PY2 else PY27 - custom_interpreter = setup_interpreter( + custom_interpreter = vendor.setup_interpreter( interpreter=PythonInterpreter.from_binary(ensure_python_interpreter(interp_version)), - interpreter_cache_dir=os.path.join(pex_root, 'interpreters'), - repos=None, # Default to PyPI. use_wheel=True ) pex_info = PexInfo.default(custom_interpreter) @@ -349,72 +345,51 @@ def test_activate_interpreter_different_from_current(): pytest.fail('PEX activation of %s failed with %s' % (pex, e)) -def test_execute_interpreter_dashc_program(): +def assert_execute_interpreter(args, expected_stdout, stdin=None, content=None): with temporary_dir() as pex_chroot: - pex_builder = PEXBuilder(path=pex_chroot) - pex_builder.freeze() - process = PEX(pex_chroot).run(args=['-c', 'import sys; print(" ".join(sys.argv))', 'one'], + with vendor.adjusted_sys_path(): + pex_builder = PEXBuilder(path=pex_chroot, interpreter=vendor.setup_interpreter()) + for rel_path, content in (content or {}).items(): + with named_temporary_file() as fp: + fp.write(content) + fp.close() + pex_builder.add_source(fp.name, rel_path) + pex_builder.freeze() + + process = PEX(pex_chroot).run(args=args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False) - stdout, stderr = process.communicate() - + stdout, stderr = process.communicate(input=stdin) assert 0 == process.returncode - assert b'-c one\n' == stdout + assert expected_stdout == stdout assert b'' == stderr -def test_execute_interpreter_dashm_module(): - with temporary_dir() as pex_chroot: - pex_builder = PEXBuilder(path=pex_chroot) - with temporary_file(root_dir=pex_chroot) as fp: - fp.write(b'import sys; print(" ".join(sys.argv))') - fp.close() - pex_builder.add_source(fp.name, 'foo/bar.py') - pex_builder.freeze() - process = PEX(pex_chroot).run(args=['-m', 'foo.bar', 'one', 'two'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - blocking=False) - stdout, stderr = process.communicate() +def test_execute_interpreter_dashc_program(): + assert_execute_interpreter(args=['-c', 'import sys; print(" ".join(sys.argv))', 'one'], + expected_stdout=b'-c one\n') - assert 0 == process.returncode - assert b'foo.bar one two\n' == stdout - assert b'' == stderr +def test_execute_interpreter_dashm_module(): + assert_execute_interpreter(content={'foo/bar.py': b'import sys; print(" ".join(sys.argv))'}, + args=['-m', 'foo.bar', 'one', 'two'], + expected_stdout=b'foo.bar one two\n') -def test_execute_interpreter_stdin_program(): - with temporary_dir() as pex_chroot: - pex_builder = PEXBuilder(path=pex_chroot) - pex_builder.freeze() - process = PEX(pex_chroot).run(args=['-', 'one', 'two'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - blocking=False) - stdout, stderr = process.communicate(input=b'import sys; print(" ".join(sys.argv))') - assert 0 == process.returncode - assert b'- one two\n' == stdout - assert b'' == stderr +def test_execute_interpreter_stdin_program(): + assert_execute_interpreter(args=['-', 'one', 'two'], + stdin=b'import sys; print(" ".join(sys.argv))', + expected_stdout=b'- one two\n') def test_execute_interpreter_file_program(): - with temporary_dir() as pex_chroot: - pex_builder = PEXBuilder(path=pex_chroot) - pex_builder.freeze() - with temporary_file(root_dir=pex_chroot) as fp: - fp.write(b'import sys; print(" ".join(sys.argv))') - fp.close() - process = PEX(pex_chroot).run(args=[fp.name, 'one', 'two'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - blocking=False) - stdout, stderr = process.communicate() - - assert 0 == process.returncode - assert '{} one two\n'.format(fp.name).encode('utf-8') == stdout - assert b'' == stderr + with named_temporary_file() as fp: + fp.write(b'import sys; print(" ".join(sys.argv))') + fp.close() + assert_execute_interpreter(args=[fp.name, 'one', 'two'], + expected_stdout='{} one two\n'.format(fp.name).encode('utf-8')) def test_pex_run_custom_setuptools_useable(): @@ -432,8 +407,8 @@ def test_pex_run_custom_setuptools_useable(): def test_pex_run_conflicting_custom_setuptools_useable(): - # Here we use an older setuptools to build the pex which has a newer setuptools requirement. - # These setuptools dists have different pkg_resources APIs: + # Here we use our vendored, newer setuptools to build the pex which has an older setuptools + # requirement. These setuptools dists have different pkg_resources APIs: # $ diff \ # <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \ # <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort) @@ -441,19 +416,29 @@ def test_pex_run_conflicting_custom_setuptools_useable(): # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py with temporary_dir() as resolve_cache: - dists = [resolved_dist.distribution - for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] - interpreter = PythonInterpreter.from_binary(sys.executable, - path_extras=[dist.location for dist in dists], - include_site_extras=False) - dists = [resolved_dist.distribution - for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)] - with temporary_dir() as temp_dir: - pex = write_simple_pex( - temp_dir, - 'from pkg_resources import appdirs, py31compat', - dists=dists, - interpreter=interpreter - ) - rc = PEX(pex.path()).run() - assert rc == 0 + with vendor.adjusted_sys_path(include_wheel=True): + dists = [resolved_dist.distribution + for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] + with temporary_dir() as temp_dir: + pex = write_simple_pex( + temp_dir, + exe_contents=textwrap.dedent(""" + import sys + import pkg_resources + + try: + from pkg_resources import appdirs + sys.exit(1) + except ImportError: + pass + + try: + from pkg_resources import py31compat + sys.exit(2) + except ImportError: + pass + """), + dists=dists, + ) + rc = PEX(pex.path()).run() + assert rc == 0 diff --git a/tests/test_pex_binary.py b/tests/test_pex_binary.py index 63f6f1810..1cb80234a 100644 --- a/tests/test_pex_binary.py +++ b/tests/test_pex_binary.py @@ -6,8 +6,7 @@ from optparse import OptionParser from tempfile import NamedTemporaryFile -from twitter.common.contextutil import temporary_dir - +from pex import vendor from pex.bin.pex import build_pex, configure_clp, configure_clp_pex_resolution from pex.common import safe_copy from pex.compatibility import to_bytes @@ -15,7 +14,7 @@ from pex.package import SourcePackage, WheelPackage from pex.resolver_options import ResolverOptionsBuilder from pex.sorter import Sorter -from pex.testing import make_sdist +from pex.testing import make_sdist, temporary_dir try: from unittest import mock @@ -185,5 +184,6 @@ def __init__(self, # # With a correct behavior the assert line is reached and pex_builder object created. with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher): - pex_builder = build_pex(reqs, options, resolver_options_builder) - assert pex_builder is not None + with vendor.adjusted_sys_path(include_wheel=True): + pex_builder = build_pex(reqs, options, resolver_options_builder) + assert pex_builder is not None diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 29b81d74e..2de2a22b7 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -4,12 +4,20 @@ import os import pytest -from twitter.common.contextutil import temporary_dir +from pex import vendor from pex.common import open_zip from pex.interpreter import PythonInterpreter from pex.pex_bootstrapper import find_compatible_interpreters, get_pex_info -from pex.testing import IS_PYPY, PY27, PY35, PY36, ensure_python_interpreter, write_simple_pex +from pex.testing import ( + IS_PYPY, + PY27, + PY35, + PY36, + ensure_python_interpreter, + temporary_dir, + write_simple_pex +) def test_get_pex_info(): @@ -40,7 +48,9 @@ def test_find_compatible_interpreters(): pex_python_path = ':'.join([py27, py35, py36]) def find_interpreters(*constraints): - return [interp.binary for interp in find_compatible_interpreters(pex_python_path, constraints)] + with vendor.adjusted_sys_path(): + return [interp.binary + for interp in find_compatible_interpreters(pex_python_path, constraints)] assert [py35, py36] == find_interpreters('>3') assert [py27] == find_interpreters('<3') @@ -54,5 +64,7 @@ def find_interpreters(*constraints): assert [] == find_interpreters('>{}, <{}'.format(PY27, PY35)) # All interpreters on PATH. - interpreters = find_compatible_interpreters(pex_python_path='', compatibility_constraints=['<3']) + with vendor.adjusted_sys_path(): + interpreters = find_compatible_interpreters(pex_python_path='', + compatibility_constraints=['<3']) assert set(interpreters).issubset(set(PythonInterpreter.all())) diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index b0843b3d0..5e5c40259 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -3,16 +3,16 @@ import os import stat +from contextlib import contextmanager import pytest -from twitter.common.contextutil import temporary_dir -from twitter.common.dirutil import safe_mkdir +from pex import vendor from pex.common import open_zip from pex.compatibility import WINDOWS, nested from pex.pex import PEX from pex.pex_builder import PEXBuilder -from pex.testing import make_bdist +from pex.testing import make_bdist, safe_mkdir, temporary_dir from pex.testing import write_simple_pex as write_pex from pex.util import DistributionHelper @@ -80,20 +80,28 @@ def test_pex_builder_wheeldep(): assert fp.read() == 'success' +@contextmanager +def pex_builder(interpreter=None, **kwargs): + with vendor.adjusted_sys_path(): + yield PEXBuilder(interpreter=vendor.setup_interpreter(interpreter), **kwargs) + + def test_pex_builder_shebang(): + @contextmanager def builder(shebang): - pb = PEXBuilder() - pb.set_shebang(shebang) - return pb + with pex_builder() as pb: + pb.set_shebang(shebang) + yield pb - for pb in builder('foobar'), builder('#!foobar'): - for b in pb, pb.clone(): - with temporary_dir() as td: - target = os.path.join(td, 'foo.pex') - b.build(target) - expected_preamble = b'#!foobar\n' - with open(target, 'rb') as fp: - assert fp.read(len(expected_preamble)) == expected_preamble + with nested(builder('foobar'), builder('#!foobar')) as (pb1, pb2): + for pb in pb1, pb2: + for b in pb, pb.clone(): + with temporary_dir() as td: + target = os.path.join(td, 'foo.pex') + b.build(target) + expected_preamble = b'#!foobar\n' + with open(target, 'rb') as fp: + assert fp.read(len(expected_preamble)) == expected_preamble def test_pex_builder_preamble(): @@ -107,8 +115,8 @@ def test_pex_builder_preamble(): "sys.exit(3)" ]) - pex_builder = PEXBuilder(preamble=tempfile_preamble) - pex_builder.build(target) + with pex_builder(preamble=tempfile_preamble) as pb: + pb.build(target) assert not os.path.exists(should_create) @@ -131,10 +139,10 @@ def test_pex_builder_compilation(): fp.write(exe_main) def build_and_check(path, precompile): - pb = PEXBuilder(path) - pb.add_source(src, 'lib/src.py') - pb.set_executable(exe, 'exe.py') - pb.freeze(bytecode_compile=precompile) + with pex_builder(path=path) as pb: + pb.add_source(src, 'lib/src.py') + pb.set_executable(exe, 'exe.py') + pb.freeze(bytecode_compile=precompile) for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'): pyc_exists = os.path.exists(os.path.join(path, pyc_file)) if precompile: diff --git a/tests/test_resolvable.py b/tests/test_resolvable.py index 3428cd46b..2805a3a44 100644 --- a/tests/test_resolvable.py +++ b/tests/test_resolvable.py @@ -4,6 +4,7 @@ import pkg_resources import pytest +from pex import vendor from pex.iterator import Iterator from pex.package import Package, SourcePackage from pex.resolvable import ( @@ -37,10 +38,10 @@ def test_resolvable_package(): assert mock_iterator.iter.mock_calls == [] assert resolvable.name == 'foo' assert resolvable.exact is True - assert resolvable.extras() == [] + assert resolvable.extras == [] resolvable = ResolvablePackage.from_string(source_name + '[extra1,extra2]', builder) - assert resolvable.extras() == ['extra1', 'extra2'] + assert resolvable.extras == ['extra1', 'extra2'] assert Resolvable.get('foo-2.3.4.tar.gz') == ResolvablePackage.from_string( 'foo-2.3.4.tar.gz', builder) @@ -62,7 +63,7 @@ def test_resolvable_requirement(): assert resolvable.requirement == pkg_resources.Requirement.parse('foo[bar]==2.3.4') assert resolvable.name == 'foo' assert resolvable.exact is True - assert resolvable.extras() == ['bar'] + assert resolvable.extras == ['bar'] assert resolvable.options._fetchers == [] assert resolvable.packages() == [] @@ -84,15 +85,16 @@ def test_resolvable_requirement(): def test_resolvable_directory(): builder = ResolverOptionsBuilder() + interpreter = vendor.setup_interpreter() with make_source_dir(name='my_project') as td: - rdir = ResolvableDirectory.from_string(td, builder) + rdir = ResolvableDirectory.from_string(td, builder, interpreter=interpreter) assert rdir.name == pkg_resources.safe_name('my_project') - assert rdir.extras() == [] + assert rdir.extras == [] - rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder) + rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder, interpreter=interpreter) assert rdir.name == pkg_resources.safe_name('my_project') - assert rdir.extras() == ['extra1', 'extra2'] + assert rdir.extras == ['extra1', 'extra2'] def test_resolvables_from_iterable(): diff --git a/tests/test_resolver.py b/tests/test_resolver.py index d6f3e3ba6..0259f310d 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -5,7 +5,6 @@ import time import pytest -from twitter.common.contextutil import temporary_dir from pex.common import safe_copy from pex.crawler import Crawler @@ -14,15 +13,21 @@ from pex.resolvable import ResolvableRequirement from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi from pex.resolver_options import ResolverOptionsBuilder -from pex.testing import make_sdist +from pex.testing import make_sdist, temporary_dir +from pex.vendor import setup_interpreter + + +def do_resolve_multi(*args, **kwargs): + kwargs.setdefault('interpreters', [setup_interpreter(use_wheel=True)]) + return list(resolve_multi(*args, **kwargs)) def test_empty_resolve(): - empty_resolve_multi = list(resolve_multi([])) + empty_resolve_multi = do_resolve_multi([]) assert empty_resolve_multi == [] with temporary_dir() as td: - empty_resolve_multi = list(resolve_multi([], cache=td)) + empty_resolve_multi = do_resolve_multi([], cache=td) assert empty_resolve_multi == [] @@ -32,7 +37,7 @@ def test_simple_local_resolve(): with temporary_dir() as td: safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist))) fetchers = [Fetcher([td])] - resolved_dists = list(resolve_multi(['project'], fetchers=fetchers)) + resolved_dists = do_resolve_multi(['project'], fetchers=fetchers) assert len(resolved_dists) == 1 @@ -46,9 +51,10 @@ def test_diamond_local_resolve_cached(): safe_copy(sdist, os.path.join(dd, os.path.basename(sdist))) fetchers = [Fetcher([dd])] with temporary_dir() as cd: - resolved_dists = list( - resolve_multi(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) - ) + resolved_dists = do_resolve_multi(['project1', 'project2'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) assert len(resolved_dists) == 2 @@ -63,24 +69,20 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): fetchers = [Fetcher([td])] with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache - resolved_dists = list( - resolve_multi(['project', 'project==1.0.0'], - fetchers=fetchers, - cache=cd, - cache_ttl=1000) - ) + resolved_dists = do_resolve_multi(['project', 'project==1.0.0'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) assert len(resolved_dists) == 1 assert resolved_dists[0].distribution.version == '1.0.0' # This simulates separate invocations of pex but allows us to keep the same tmp cache dir Crawler.reset_cache() # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td - resolved_dists = list( - resolve_multi(['project', 'project==1.1.0'], - fetchers=fetchers, - cache=cd, - cache_ttl=1000) - ) + resolved_dists = do_resolve_multi(['project', 'project==1.1.0'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) assert len(resolved_dists) == 1 assert resolved_dists[0].distribution.version == '1.1.0' # Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact @@ -88,12 +90,10 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): # resolvable_set.merge() would fail. Crawler.reset_cache() time.sleep(1) - resolved_dists = list( - resolve_multi(['project', 'project==1.1.0'], - fetchers=fetchers, - cache=cd, - cache_ttl=1) - ) + resolved_dists = do_resolve_multi(['project', 'project==1.1.0'], + fetchers=fetchers, + cache=cd, + cache_ttl=1) assert len(resolved_dists) == 1 assert resolved_dists[0].distribution.version == '1.1.0' @@ -110,12 +110,10 @@ def test_ambiguous_transitive_resolvable(): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: - resolved_dists = list( - resolve_multi(['foo', 'bar'], - fetchers=fetchers, - cache=cd, - cache_ttl=1000) - ) + resolved_dists = do_resolve_multi(['foo', 'bar'], + fetchers=fetchers, + cache=cd, + cache_ttl=1000) assert len(resolved_dists) == 2 assert resolved_dists[0].distribution.version == '1.0.0' @@ -130,9 +128,7 @@ def test_resolve_prereleases(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - resolved_dists = list( - resolve_multi(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) - ) + resolved_dists = do_resolve_multi(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) assert 1 == len(resolved_dists) resolved_dist = resolved_dists[0] assert expected_version == resolved_dist.distribution.version @@ -153,9 +149,7 @@ def test_resolve_prereleases_cached(): with temporary_dir() as cd: def assert_resolve(dep, expected_version, **resolve_kwargs): - resolved_dists = list( - resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs) - ) + resolved_dists = do_resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs) assert 1 == len(resolved_dists) resolved_dist = resolved_dists[0] assert expected_version == resolved_dist.distribution.version @@ -185,9 +179,7 @@ def test_resolve_prereleases_and_no_version(): fetchers = [Fetcher([td])] def assert_resolve(deps, expected_version, **resolve_kwargs): - resolved_dists = list( - resolve_multi(deps, fetchers=fetchers, **resolve_kwargs) - ) + resolved_dists = do_resolve_multi(deps, fetchers=fetchers, **resolve_kwargs) assert 1 == len(resolved_dists) resolved_dist = resolved_dists[0] assert expected_version == resolved_dist.distribution.version @@ -216,11 +208,9 @@ def test_resolve_prereleases_multiple_set(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - resolved_dists = list( - resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'], - fetchers=fetchers, - **resolve_kwargs) - ) + resolved_dists = do_resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'], + fetchers=fetchers, + **resolve_kwargs) assert 1 == len(resolved_dists) resolved_dist = resolved_dists[0] assert expected_version == resolved_dist.distribution.version diff --git a/tests/test_util.py b/tests/test_util.py index 54e0b309d..95993118f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -6,13 +6,12 @@ from hashlib import sha1 from textwrap import dedent -from twitter.common.contextutil import temporary_dir - +from pex import vendor from pex.common import open_zip, safe_mkdir from pex.compatibility import nested, to_bytes from pex.installer import EggInstaller, WheelInstaller from pex.pex_builder import PEXBuilder -from pex.testing import make_bdist, run_simple_pex, temporary_content, write_zipfile +from pex.testing import make_bdist, run_simple_pex, temporary_content, temporary_dir, write_zipfile from pex.util import ( CacheHelper, DistributionHelper, @@ -135,20 +134,22 @@ def test_access_zipped_assets_integration(): print(line) ''') with nested(temporary_dir(), temporary_dir()) as (td1, td2): - pb = PEXBuilder(path=td1) - with open(os.path.join(td1, 'exe.py'), 'w') as fp: - fp.write(test_executable) - pb.set_executable(fp.name) - - submodule = os.path.join(td1, 'my_package', 'submodule') - safe_mkdir(submodule) - mod_path = os.path.join(submodule, 'mod.py') - with open(mod_path, 'w') as fp: - fp.write('accessed') - pb.add_source(fp.name, 'my_package/submodule/mod.py') - - pex = os.path.join(td2, 'app.pex') - pb.build(pex) + # PEXBuilder uses pkg_resources. + with vendor.adjusted_sys_path(): + pb = PEXBuilder(path=td1, interpreter=vendor.setup_interpreter()) + with open(os.path.join(td1, 'exe.py'), 'w') as fp: + fp.write(test_executable) + pb.set_executable(fp.name) + + submodule = os.path.join(td1, 'my_package', 'submodule') + safe_mkdir(submodule) + mod_path = os.path.join(submodule, 'mod.py') + with open(mod_path, 'w') as fp: + fp.write('accessed') + pb.add_source(fp.name, 'my_package/submodule/mod.py') + + pex = os.path.join(td2, 'app.pex') + pb.build(pex) output, returncode = run_simple_pex(pex) try: diff --git a/tox.ini b/tox.ini index 079783b4c..000adb342 100644 --- a/tox.ini +++ b/tox.ini @@ -9,16 +9,14 @@ commands = py.test --ignore="tests/test_integration.py" {posargs:-vvs} # Ensure pex's main entrypoint can be run externally. - pex --cache-dir {envtmpdir}/buildcache wheel requests . -e pex.bin.pex:main --version + pex --cache-dir {envtmpdir}/buildcache . -e pex.bin.pex:main --version deps = pytest==3.7.2 twitter.common.contextutil>=0.3.1,<0.4.0 twitter.common.dirutil>=0.3.1,<0.4.0 twitter.common.lang>=0.3.1,<0.4.0 twitter.common.testing>=0.3.1,<0.4.0 - wheel==0.31.1 packaging==16.8 - setuptools<34.0 list-tests: mock==2.0.0 py27: mock==2.0.0 pypy: mock==2.0.0 @@ -29,7 +27,9 @@ deps = cachecontrol: lockfile coverage: coverage==4.5.1 subprocess: subprocess32 -whitelist_externals = open +whitelist_externals = + open + bash [testenv:integration-tests] deps = @@ -124,17 +124,33 @@ basepython = python2.7 deps = twitter.checkstyle commands = - twitterstyle -n ImportOrder {toxinidir}/pex {toxinidir}/tests + bash scripts/style.sh [testenv:isort-run] basepython = python2.7 deps = isort -commands = isort -ns __init__.py -rc {toxinidir}/pex {toxinidir}/tests +commands = + isort \ + -rc \ + -ns __init__.py \ + -sg {toxinidir}/pex/vendor/_vendored/** \ + -rc \ + {toxinidir}/pex {toxinidir}/tests [testenv:isort-check] basepython = python2.7 deps = isort -commands = isort -ns __init__.py -rc -c {toxinidir}/pex {toxinidir}/tests +commands = + isort \ + -c \ + -rc \ + -ns __init__.py \ + -sg {toxinidir}/pex/vendor/_vendored/** \ + {toxinidir}/pex {toxinidir}/tests + +[testenv:vendor] +deps = pip==18.1 +commands = python -m pex.vendor [testenv:docs] changedir = docs @@ -155,19 +171,19 @@ commands = pex {posargs:} commands = pex {posargs:} [testenv:py27-package] -commands = pex --cache-dir {envtmpdir}/buildcache wheel requests . -o dist/pex27 -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache requests . -o dist/pex27 -e pex.bin.pex:main -v [testenv:py34-package] -commands = pex --cache-dir {envtmpdir}/buildcache wheel requests . -o dist/pex34 -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache requests . -o dist/pex34 -e pex.bin.pex:main -v [testenv:py35-package] -commands = pex --cache-dir {envtmpdir}/buildcache wheel requests . -o dist/pex35 -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache requests . -o dist/pex35 -e pex.bin.pex:main -v [testenv:py36-package] -commands = pex --cache-dir {envtmpdir}/buildcache wheel requests . -o dist/pex36 -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache requests . -o dist/pex36 -e pex.bin.pex:main -v [testenv:py37-package] -commands = pex --cache-dir {envtmpdir}/buildcache wheel requests . -o dist/pex37 -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache requests . -o dist/pex37 -e pex.bin.pex:main -v # Would love if you didn't have to enumerate environments here :-\ [testenv:py27]