From c93ea9c7dacbc8eed201d72c721d39e4c01897cf Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 14 Apr 2015 14:14:03 -0400 Subject: [PATCH] Remove -s option and allow for PEXing directories directly. This is a backwards incompatible command line change with pre-1.x as it removes the -s flag. --- CHANGES.rst | 9 +++++++ pex/bin/pex.py | 22 ++------------- pex/iterator.py | 2 +- pex/resolvable.py | 58 ++++++++++++++++++++++++++++++++++++---- pex/testing.py | 7 +++++ pex/version.py | 2 +- scripts/coverage.sh | 2 +- tests/test_resolvable.py | 29 ++++++++++++++++---- tox.ini | 4 +-- 9 files changed, 100 insertions(+), 35 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 042ea953e..abff608fe 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,15 @@ CHANGES ======= +---------- +1.0.0.dev2 +---------- + +* Now supports extras for static URLs and installable directories. + +* BREAKING CHANGE: Removes the ``-s`` option in favor of specifying directories directly as + arguments to the pex command line. + ---------- 1.0.0.dev1 ---------- diff --git a/pex/bin/pex.py b/pex/bin/pex.py index efab03650..df5769ed7 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -20,7 +20,7 @@ from pex.crawler import Crawler from pex.fetcher import Fetcher, PyPIFetcher from pex.http import Context -from pex.installer import EggInstaller, InstallerBase, Packager +from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage @@ -28,7 +28,7 @@ from pex.pex_builder import PEXBuilder from pex.platforms import Platform from pex.requirements import requirements_from_file -from pex.resolvable import Resolvable, ResolvablePackage +from pex.resolvable import Resolvable from pex.resolver import CachingResolver, Resolver from pex.resolver_options import ResolverOptionsBuilder from pex.tracer import TRACER, TraceLogger @@ -300,15 +300,6 @@ def configure_clp(): help='Add requirements from the given requirements file. This option can be used multiple ' 'times.') - parser.add_option( - '-s', '--source-dir', - dest='source_dirs', - metavar='DIR', - default=[], - action='append', - help='Source to be packaged; This should be a pip-installable project ' - 'with a setup.py.') - parser.add_option( '-v', dest='verbosity', @@ -432,15 +423,6 @@ def build_pex(args, options, resolver_option_builder): for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) - if options.source_dirs: - for source_dir in options.source_dirs: - try: - sdist = Packager(source_dir, interpreter=interpreter).sdist() - except InstallerBase.Error: - die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) - - resolvables.append(ResolvablePackage.from_string(sdist, resolver_option_builder)) - resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: diff --git a/pex/iterator.py b/pex/iterator.py index bc733f17d..2451cc489 100644 --- a/pex/iterator.py +++ b/pex/iterator.py @@ -22,7 +22,7 @@ class Iterator(IteratorInterface): def __init__(self, fetchers=None, crawler=None, follow_links=False): self._crawler = crawler or Crawler() - self._fetchers = fetchers or [PyPIFetcher()] + self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()] self.__follow_links = follow_links def _iter_requirement_urls(self, req): diff --git a/pex/resolvable.py b/pex/resolvable.py index 54b8583d7..81cf78d47 100644 --- a/pex/resolvable.py +++ b/pex/resolvable.py @@ -1,17 +1,35 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import os +import re from abc import abstractmethod, abstractproperty -from pkg_resources import Requirement +from pkg_resources import Requirement, safe_extra from .base import maybe_requirement, requirement_is_exact from .compatibility import string as compatibility_string from .compatibility import AbstractClass +from .installer import InstallerBase, Packager from .package import Package from .resolver_options import ResolverOptionsBuilder, ResolverOptionsInterface +# Extract extras as specified per "declaring extras": +# https://pythonhosted.org/setuptools/setuptools.html +_EXTRAS_PATTERN = re.compile(r'(?P
.*)\[(?P.*)\]$') + + +def strip_extras(resolvable_string): + match = _EXTRAS_PATTERN.match(resolvable_string) + if match: + resolvable_string, extras = match.groupdict()['main'], match.groupdict()['extras'] + extras = [safe_extra(extra.strip()) for extra in extras.split(',')] + else: + extras = [] + return resolvable_string, extras + + class Resolvable(AbstractClass): """An entity that can be resolved into a package.""" @@ -136,13 +154,15 @@ class ResolvablePackage(Resolvable): # TODO(wickman) Implement extras parsing for ResolvablePackage @classmethod def from_string(cls, requirement_string, options_builder): + requirement_string, extras = strip_extras(requirement_string) package = Package.from_href(requirement_string) if package is None: raise cls.InvalidRequirement('Requirement string does not appear to be a package.') - return cls(package, options_builder.build(package.name)) + return cls(package, options_builder.build(package.name), extras=extras) - def __init__(self, package, options): + def __init__(self, package, options, extras=None): self.package = package + self._extras = extras super(ResolvablePackage, self).__init__(options) def compatible(self, iterator): @@ -159,9 +179,8 @@ def name(self): def exact(self): return True - # TODO(wickman) Implement extras parsing for ResolvablePackages def extras(self, interpreter=None): - return [] + return self._extras def __eq__(self, other): return isinstance(other, ResolvablePackage) and self.package == other.package @@ -219,6 +238,35 @@ def __str__(self): return str(self.requirement) +class ResolvableDirectory(ResolvablePackage): + """A source directory (with setup.py) resolvable.""" + + @classmethod + def is_installable(cls, requirement_string): + if not os.path.isdir(requirement_string): + return False + return os.path.isfile(os.path.join(requirement_string, 'setup.py')) + + @classmethod + def from_string(cls, requirement_string, options_builder): + requirement_string, extras = strip_extras(requirement_string) + if cls.is_installable(requirement_string): + try: + # TODO(wickman) This is one case where interpreter is necessary to be fully correct. This + # may indicate that packages() should take interpreter like extras does. Once we have + # metadata in setup.cfg or whatever, then we can get the interpreter out of the equation. + sdist = Packager(requirement_string).sdist() + except InstallerBase.Error: + raise cls.InvalidRequirement('Could not create source distribution for %s' % + requirement_string) + package = Package.from_href(sdist) + return ResolvablePackage(package, options_builder.build(package.name), extras=extras) + else: + raise cls.InvalidRequirement('%s does not appear to be an installable directory.' + % requirement_string) + + +Resolvable.register(ResolvableDirectory) Resolvable.register(ResolvableRepository) Resolvable.register(ResolvablePackage) Resolvable.register(ResolvableRequirement) diff --git a/pex/testing.py b/pex/testing.py index 034c72020..52e185a71 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -92,6 +92,13 @@ def make_installer(name='my_project', installer_impl=EggInstaller, zip_safe=True yield installer_impl(td) +@contextlib.contextmanager +def make_source_dir(name='my_project'): + interp = {'project_name': name, 'zip_safe': True} + with temporary_content(PROJECT_CONTENT, interp=interp) as td: + yield td + + def make_sdist(name='my_project', zip_safe=True): with make_installer(name=name, installer_impl=Packager, zip_safe=zip_safe) as packager: return packager.sdist() diff --git a/pex/version.py b/pex/version.py index 47c5cf188..8b56e9041 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.0.dev1' +__version__ = '1.0.0.dev2' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 9dba64775..680676cb0 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -4,4 +4,4 @@ coverage run -p -m py.test tests coverage run -p -m pex.bin.pex -v --help >&/dev/null coverage run -p -m pex.bin.pex -v -- scripts/do_nothing.py coverage run -p -m pex.bin.pex -v requests -- scripts/do_nothing.py -coverage run -p -m pex.bin.pex -v -s . setuptools -- scripts/do_nothing.py +coverage run -p -m pex.bin.pex -v . setuptools -- scripts/do_nothing.py diff --git a/tests/test_resolvable.py b/tests/test_resolvable.py index 0f0f3f099..ef4c07388 100644 --- a/tests/test_resolvable.py +++ b/tests/test_resolvable.py @@ -8,12 +8,14 @@ from pex.package import Package, SourcePackage from pex.resolvable import ( Resolvable, + ResolvableDirectory, ResolvablePackage, ResolvableRepository, ResolvableRequirement, resolvables_from_iterable ) from pex.resolver_options import ResolverOptionsBuilder +from pex.testing import make_source_dir try: from unittest import mock @@ -22,9 +24,10 @@ def test_resolvable_package(): + builder = ResolverOptionsBuilder() source_name = 'foo-2.3.4.tar.gz' pkg = SourcePackage.from_href(source_name) - resolvable = ResolvablePackage.from_string(source_name, ResolverOptionsBuilder()) + resolvable = ResolvablePackage.from_string(source_name, builder) assert resolvable.packages() == [pkg] mock_iterator = mock.create_autospec(Iterator, spec_set=True) @@ -34,14 +37,16 @@ def test_resolvable_package(): assert mock_iterator.iter.mock_calls == [] assert resolvable.name == 'foo' assert resolvable.exact is True - # TODO(wickman) Implement extras parsing for resolvable packages. assert resolvable.extras() == [] + resolvable = ResolvablePackage.from_string(source_name + '[extra1,extra2]', builder) + assert resolvable.extras() == ['extra1', 'extra2'] + assert Resolvable.get('foo-2.3.4.tar.gz') == ResolvablePackage.from_string( - 'foo-2.3.4.tar.gz', ResolverOptionsBuilder()) + 'foo-2.3.4.tar.gz', builder) with pytest.raises(ResolvablePackage.InvalidRequirement): - ResolvablePackage.from_string('foo', ResolverOptionsBuilder()) + ResolvablePackage.from_string('foo', builder) def test_resolvable_repository(): @@ -53,11 +58,12 @@ def test_resolvable_repository(): def test_resolvable_requirement(): req = 'foo[bar]==2.3.4' - resolvable = ResolvableRequirement.from_string(req, ResolverOptionsBuilder()) + resolvable = ResolvableRequirement.from_string(req, ResolverOptionsBuilder(fetchers=[])) assert resolvable.requirement == pkg_resources.Requirement.parse('foo[bar]==2.3.4') assert resolvable.name == 'foo' assert resolvable.exact is True assert resolvable.extras() == ['bar'] + assert resolvable.options._fetchers == [] assert resolvable.packages() == [] source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz') @@ -76,6 +82,19 @@ def test_resolvable_requirement(): 'foo', ResolverOptionsBuilder()) +def test_resolvable_directory(): + builder = ResolverOptionsBuilder() + + with make_source_dir(name='my_project') as td: + rdir = ResolvableDirectory.from_string(td, builder) + assert rdir.name == pkg_resources.safe_name('my_project') + assert rdir.extras() == [] + + rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder) + assert rdir.name == pkg_resources.safe_name('my_project') + assert rdir.extras() == ['extra1', 'extra2'] + + def test_resolvables_from_iterable(): builder = ResolverOptionsBuilder() diff --git a/tox.ini b/tox.ini index a3bd5324f..bb0f47b87 100644 --- a/tox.ini +++ b/tox.ini @@ -111,10 +111,10 @@ commands = pex {posargs:} commands = pex {posargs:} [testenv:py27-package] -commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests -s . -o dist/pex -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests . -o dist/pex -e pex.bin.pex:main -v [testenv:py34-package] -commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests -s . -o dist/pex -e pex.bin.pex:main -v +commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests . -o dist/pex -e pex.bin.pex:main -v # Would love if you didn't have to enumerate environments here :-\ [testenv:py26]