diff --git a/CHANGES.rst b/CHANGES.rst index 12b445d47..77d922ca4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ CHANGES 1.1.0.dev0 ---------- +* Adds the ``bdist_pex`` command to setuptools. + `#99 `_. + * Bug fix: We did not normalize package names in ``ResolvableSet``, so it was possible to depend on ``sphinx`` and ``Sphinx-1.4a0.tar.gz`` and get two versions build and included into the pex. `#147 `_. @@ -16,22 +19,22 @@ CHANGES 1.0.3 ----- -* Bug fix: Accommodate OSX `Python` python binaries. Previously the OSX python distributions shipped +* Bug fix: Accommodate OSX ``Python`` python binaries. Previously the OSX python distributions shipped with OSX, XCode and available via https://www.python.org/downloads/ could fail to be detected using - the `PythonInterpreter` class. + the ``PythonInterpreter`` class. Fixes `#144 `_. * Bug fix: PEX_SCRIPT failed when the script was from a not-zip-safe egg. Original PR `#139 `_. -* Bug fix: `sys.exit` called without arguments would cause `None` to be printed on stderr since pex 1.0.1. +* Bug fix: ``sys.exit`` called without arguments would cause `None` to be printed on stderr since pex 1.0.1. `#143 `_. ----- 1.0.2 ----- -* Bug fix: PEX-INFO values were overridden by environment `Variables` with default values that were +* Bug fix: PEX-INFO values were overridden by environment ``Variables`` with default values that were not explicitly set in the environment. Fixes `#135 `_. diff --git a/docs/buildingpex.rst b/docs/buildingpex.rst index 8a16acfee..dcbf19915 100644 --- a/docs/buildingpex.rst +++ b/docs/buildingpex.rst @@ -1,12 +1,13 @@ .. _buildingpex: -******************* Building .pex files -******************* +=================== The easiest way to build .pex files is with the ``pex`` utility, which is made available when you ``pip install pex``. Do this within a virtualenv, then you can use -pex to bootstrap itself:: +pex to bootstrap itself: + +.. code-block:: bash $ pex pex requests -c pex -o ~/bin/pex @@ -15,12 +16,25 @@ console script named "pex", saving it in ~/bin/pex. At this point, assuming ~/bin is on your $PATH, then you can use pex in or outside of any virtualenv. +The second easiest way to build .pex files is using the ``bdist_pex`` setuptools command +which is available if you ``pip install pex``. For example, to clone and build pip from source: + +.. code-block:: bash + + $ git clone https://github.com/pypa/pip && cd pip + $ python setup.py bdist_pex + running bdist_pex + Writing pip to dist/pip-7.2.0.dev0.pex + +Both are described in more detail below. Invoking the ``pex`` utility ----------------------------- +============================ The ``pex`` utility has no required arguments and by default will construct an empty environment -and invoke it. When no entry point is specified, "invocation" means starting an interpreter:: +and invoke it. When no entry point is specified, "invocation" means starting an interpreter: + +.. code-block:: bash $ pex Python 2.6.9 (unknown, Jan 2 2014, 14:52:48) @@ -33,7 +47,9 @@ This creates an ephemeral environment that only exists for the duration of the ` and is garbage collected immediately on exit. You can tailor which interpreter is used by specifying ``--python=PATH``. PATH can be either the -absolute path of a Python binary or the name of a Python interpreter within the environment, e.g.:: +absolute path of a Python binary or the name of a Python interpreter within the environment, e.g.: + +.. code-block:: bash $ pex --python=python3.3 Python 3.3.3 (default, Jan 2 2014, 14:57:01) @@ -53,7 +69,9 @@ Specifying requirements Requirements are specified using the same form as expected by ``pip`` and ``setuptools``, e.g. ``flask``, ``setuptools==2.1.2``, ``Django>=1.4,<1.6``. These are specified as arguments to pex and any number (including 0) may be specified. For example, to start an environment with ``flask`` -and ``psutil>1``:: +and ``psutil>1``: + +.. code-block:: bash $ pex flask 'psutil>1' Python 2.6.9 (unknown, Jan 2 2014, 14:52:48) @@ -62,14 +80,18 @@ and ``psutil>1``:: (InteractiveConsole) >>> -You can then import and manipulate modules like you would otherwise:: +You can then import and manipulate modules like you would otherwise: + +.. code-block:: bash >>> import flask >>> import psutil >>> ... Requirements can also be specified using the requirements.txt format, using ``pex -r``. This can be a handy -way to freeze a virtualenv into a PEX file:: +way to freeze a virtualenv into a PEX file: + +.. code-block:: bash $ pex -r <(pip freeze) -o my_application.pex @@ -80,10 +102,12 @@ Specifying entry points Entry points define how the environment is executed and may be specified in one of three ways. pex -- script.py -^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~ As mentioned above, if no entry points are specified, the default behavior is to emulate an -interpreter. First we create a simple flask application:: +interpreter. First we create a simple flask application: + +.. code-block:: bash $ cat < flask_hello_world.py > from flask import Flask @@ -96,17 +120,21 @@ interpreter. First we create a simple flask application:: > app.run() > EOF -Then, like an interpreter, if a source file is specified as a parameter to pex, it is invoked:: +Then, like an interpreter, if a source file is specified as a parameter to pex, it is invoked: + +.. code-block:: bash $ pex flask -- ./flask_hello_world.py * Running on http://127.0.0.1:5000/ pex -m -^^^^^^ +~~~~~~ Your code may be within the PEX file or it may be some predetermined entry point within the standard library. ``pex -m`` behaves very similarly to ``python -m``. Consider -``python -m pydoc``:: +``python -m pydoc``: + +.. code-block:: bash $ python -m pydoc pydoc - the Python documentation tool @@ -117,7 +145,9 @@ within the standard library. ``pex -m`` behaves very similarly to ``python -m`` reference to a class or function within a module or module in a ... -This can be emulated using the ``pex`` tool using ``-m pydoc``:: +This can be emulated using the ``pex`` tool using ``-m pydoc``: + +.. code-block:: bash $ pex -m pydoc pydoc - the Python documentation tool @@ -129,7 +159,9 @@ This can be emulated using the ``pex`` tool using ``-m pydoc``:: ... Arguments will be passed unescaped following ``--`` on the command line. So in order to -get pydoc help on the ``flask.app`` package in Flask:: +get pydoc help on the ``flask.app`` package in Flask: + +.. code-block:: bash $ pex flask -m pydoc -- flask.app @@ -151,7 +183,9 @@ Entry points can also take the form ``package:target``, such as ``sphinx:main`` and Fabric respectively. This is roughly equivalent to running a script that does ``from package import target; target()``. This can be a powerful way to invoke Python applications without ever having to ``pip install`` -anything, for example a one-off invocation of Sphinx with the readthedocs theme available:: +anything, for example a one-off invocation of Sphinx with the readthedocs theme available: + +.. code-block:: bash $ pex sphinx sphinx_rtd_theme -e sphinx:main -- --help Sphinx v1.2.2 @@ -165,11 +199,13 @@ anything, for example a one-off invocation of Sphinx with the readthedocs theme ... pex -c -^^^^^^ +~~~~~~ If you don't know the ``package:target`` for the console scripts of your favorite python packages, pex allows you to use ``-c`` to specify a console script as defined -by the distribution. For example, Fabric provides the ``fab`` tool when pip installed:: +by the distribution. For example, Fabric provides the ``fab`` tool when pip installed: + +.. code-block:: bash $ pex Fabric -c fab -- --help Fatal error: Couldn't find any fabfiles! @@ -178,7 +214,9 @@ by the distribution. For example, Fabric provides the ``fab`` tool when pip ins Aborting. -Even scripts defined by the "scripts" section of a distribution can be used, e.g. with boto:: +Even scripts defined by the "scripts" section of a distribution can be used, e.g. with boto: + +.. code-block:: bash $ pex boto -c mturk usage: mturk [-h] [-P] [--nicknames PATH] @@ -194,16 +232,22 @@ Each of the commands above have been manipulating ephemeral PEX environments -- exist for the duration of the pex command lifetime and immediately garbage collected. If the ``-o PATH`` option is specified, a PEX file of the environment is saved to disk at ``PATH``. For example -we can package a standalone Sphinx as above:: +we can package a standalone Sphinx as above: + +.. code-block:: bash $ pex sphinx sphinx_rtd_theme -c sphinx -o sphinx.pex -Instead of executing the environment, it is saved to disk:: +Instead of executing the environment, it is saved to disk: + +.. code-block:: bash $ ls -l sphinx.pex -rwxr-xr-x 1 wickman wheel 4988494 Mar 11 17:48 sphinx.pex -This is an executable environment and can be executed as before:: +This is an executable environment and can be executed as before: + +.. code-block:: bash $ ./sphinx.pex --help Sphinx v1.2.2 @@ -219,16 +263,22 @@ This is an executable environment and can be executed as before:: As before, entry points are not required, and if not specified the PEX will default to just dropping into an interpreter. If an alternate interpreter is specified with ``--python``, e.g. pypy, it will be the -default hashbang in the PEX file:: +default hashbang in the PEX file: + +.. code-block:: bash $ pex --python=pypy flask -o flask-pypy.pex -The hashbang of the PEX file specifies PyPy:: +The hashbang of the PEX file specifies PyPy: + +.. code-block:: bash $ head -1 flask-pypy.pex #!/usr/bin/env pypy -and when invoked uses the environment PyPy:: +and when invoked uses the environment PyPy: + +.. code-block:: bash $ ./flask-pypy.pex Python 2.7.3 (87aa9de10f9c, Nov 24 2013, 20:57:21) @@ -238,7 +288,9 @@ and when invoked uses the environment PyPy:: >>> import flask To specify an explicit Python shebang line (e.g. from a non-standard location or not on $PATH), -you can use the ``--python-shebang`` option:: +you can use the ``--python-shebang`` option: + +.. code-block:: bash $ dist/pex --python-shebang='/Users/wickman/Python/CPython-3.4.2/bin/python3.4' -o my.pex $ head -1 my.pex @@ -253,11 +305,13 @@ Tailoring requirement resolution In general, ``pex`` honors the same options as pip when it comes to resolving packages. Like pip, by default ``pex`` fetches artifacts from PyPI. This can be disabled with ``--no-index``. -If PyPI fetching is disabled, you will need to specify a search repository via ``-f/--find-links``. +If PyPI fetching is disabled, you will need to specify a search repository via ``-f/--find-links``. This may be a directory on disk or a remote simple http server. For example, you can delegate artifact fetching and resolution to ``pip wheel`` for whatever -reason -- perhaps you're running a firewalled mirror -- but continue to package with pex:: +reason -- perhaps you're running a firewalled mirror -- but continue to package with pex: + +.. code-block:: bash $ pip wheel -w /tmp/wheelhouse sphinx sphinx_rtd_theme $ pex -f /tmp/wheelhouse --no-index -e sphinx:main -o sphinx.pex sphinx sphinx_rtd_theme @@ -272,9 +326,9 @@ that can be used to override the runtime behavior. ``--zip-safe``/``--not-zip-safe`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Whether or not to treat the environment as zip-safe. By default PEX files are listed as zip safe. +Whether or not to treat the environment as zip-safe. By default PEX files are listed as zip safe. If ``--not-zip-safe`` is specified, the source of the PEX will be written to disk prior to invocation rather than imported via the zipimporter. NOTE: Distribution zip-safe bits will still be honored even if the PEX is marked as zip-safe. For example, included .eggs may be marked as @@ -283,14 +337,14 @@ and written to disk prior to PEX invocation. ``--not-zip-safe`` forces ``--alwa ``--always-write-cache`` -^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~ Always write all packaged dependencies within the PEX to disk prior to invocation. This forces the zip-safe bit of any dependency to be ignored. ``--inherit-path`` -^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~ By default, PEX environments are completely scrubbed empty of any packages installed on the global site path. Setting ``--inherit-path`` allows packages within site-packages to be considered as candidate distributions @@ -300,7 +354,7 @@ if a package does not package correctly an an egg or wheel.) ``--ignore-errors`` -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~ If not all of the PEX environment's dependencies resolve correctly (e.g. you are overriding the current Python interpreter with ``PEX_PYTHON``) this forces the PEX file to execute despite this. Can be useful @@ -308,7 +362,7 @@ in certain situations when particular extensions may not be necessary to run a p ``--platform`` -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~ The platform to build the pex for. Right now it defaults to the current system, but you can specify something like ``linux-x86_64`` or ``macosx-10.6-x86_64``. This will look for bdists for the particular platform. @@ -322,8 +376,52 @@ The source of truth for these environment variables can be found in the `pex.variables API `_. +Using ``bdist_pex`` +=================== + +pex provides a convenience command for use in setuptools. ``python setup.py +bdist_pex`` is a simple way to build executables for Python projects that +adhere to standard naming conventions. + +``bdist_pex`` +------------- + +The default behavior of ``bdist_pex`` is to build an executable using the +console script of the same name as the package. For example, pip has three +entry points: ``pip``, ``pip2`` and ``pip2.7`` if you're using Python 2.7. Since +there exists an entry point named ``pip`` in the ``console_scripts`` section +of the entry points, that entry point is chosen and an executable pex is produced. The pex file +will have the version number appended, e.g. ``pip-7.2.0.pex``. + +If no console scripts are provided, or the only console scripts available do +not bear the same name as the package, then an environment pex will be +produced. An environment pex is a pex file that drops you into an +interpreter with all necessary dependencies but stops short of invoking a +specific module or function. + +``bdist_pex --bdist-all`` +------------------------- + +If you would like to build all the console scripts defined in the package instead of +just the namesake script, ``--bdist-all`` will write all defined entry_points but omit +version numbers and the ``.pex`` suffix. This can be useful if you would like to +virtually install a Python package somewhere on your ``$PATH`` without doing something +scary like ``sudo pip install``: + +.. code-block:: bash + + $ git clone https://github.com/sphinx-doc/sphinx && cd sphinx + $ python setup.py bist_pex --bdist-all --bdist-dir=$HOME/bin + running bdist_pex + Writing sphinx-apidoc to /Users/wickman/bin/sphinx-apidoc + Writing sphinx-build to /Users/wickman/bin/sphinx-build + Writing sphinx-quickstart to /Users/wickman/bin/sphinx-quickstart + Writing sphinx-autogen to /Users/wickman/bin/sphinx-autogen + $ sphinx-apidoc --help | head -1 + Usage: sphinx-apidoc [options] -o [exclude_path, ...] + Other ways to build PEX files ------------------------------ +============================= There are other supported ways to build pex files: * Using pants. See `Pants Python documentation `_. diff --git a/pex/commands/__init__.py b/pex/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pex/commands/bdist_pex.py b/pex/commands/bdist_pex.py new file mode 100644 index 000000000..4f7950fee --- /dev/null +++ b/pex/commands/bdist_pex.py @@ -0,0 +1,85 @@ +import os +from distutils import log + +from setuptools import Command + +from pex.bin.pex import build_pex, configure_clp +from pex.common import die +from pex.variables import ENV + + +# Suppress checkstyle violations due to setuptools command requirements. +class bdist_pex(Command): # noqa + description = "create a PEX file from a source distribution" # noqa + + user_options = [ # noqa + ('bdist-all', None, 'pexify all defined entry points'), + ('bdist-dir=', None, 'the directory into which pexes will be written, default: dist.'), + ('pex-args=', None, 'additional arguments to the pex tool'), + ] + + boolean_options = [ # noqa + 'bdist-all', + ] + + def initialize_options(self): + self.bdist_all = False + self.bdist_dir = None + self.pex_args = '' + + def finalize_options(self): + self.pex_args = self.pex_args.split() + + def _write(self, pex_builder, target, script=None): + builder = pex_builder.clone() + + if script is not None: + builder.set_script(script) + + builder.build(target) + + def run(self): + name = self.distribution.get_name() + version = self.distribution.get_version() + parser, options_builder = configure_clp() + package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( + self.distribution.script_name))) + + if self.bdist_dir is None: + self.bdist_dir = os.path.join(package_dir, 'dist') + + options, reqs = parser.parse_args(self.pex_args) + + if options.entry_point or options.script: + die('Must not specify entry_point or script to --pex-args') + + reqs = [package_dir] + reqs + + with ENV.patch(PEX_VERBOSE=str(options.verbosity)): + pex_builder = build_pex(reqs, options, options_builder) + + def split_and_strip(entry_point): + console_script, entry_point = entry_point.split('=', 2) + return console_script.strip(), entry_point.strip() + + try: + console_scripts = dict(split_and_strip(script) + for script in self.distribution.entry_points.get('console_scripts', [])) + except ValueError: + console_scripts = {} + + if self.bdist_all: + # Write all entry points into unversioned pex files. + for script_name in console_scripts: + target = os.path.join(self.bdist_dir, script_name) + log.info('Writing %s to %s' % (script_name, target)) + self._write(pex_builder, target, script=script_name) + elif name in console_scripts: + # The package has a namesake entry point, so use it. + target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') + log.info('Writing %s to %s' % (name, target)) + self._write(pex_builder, target, script=name) + else: + # The package has no namesake entry point, so build an environment pex. + log.info('Writing environment pex into %s' % target) + self._write(pex_builder, target, script=None) diff --git a/pex/installer.py b/pex/installer.py index 17f497055..acb0bf0c6 100644 --- a/pex/installer.py +++ b/pex/installer.py @@ -190,7 +190,7 @@ def distribution(self): class DistributionPackager(InstallerBase): def mixins(self): mixins = super(DistributionPackager, self).mixins().copy() - mixins.update(setuptools='setuptools>=1') + mixins.update(setuptools=SETUPTOOLS_REQUIREMENT) return mixins def find_distribution(self): diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 5410fc95b..064112554 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -110,8 +110,13 @@ def clone(self, into=None): interpreter exit. """ chroot_clone = self._chroot.clone(into=into) - return self.__class__( - chroot=chroot_clone, interpreter=self._interpreter, pex_info=self._pex_info.copy()) + clone = self.__class__( + chroot=chroot_clone, + interpreter=self._interpreter, + pex_info=self._pex_info.copy()) + for dist in self._distributions: + clone.add_distribution(dist) + return clone def path(self): return self.chroot().path() @@ -204,7 +209,9 @@ def set_script(self, script): self._pex_info.script = script return - raise self.InvalidExecutableSpecification('Could not find script %s in PEX!' % script) + raise self.InvalidExecutableSpecification( + 'Could not find script %r in any distribution %s within PEX!' % ( + script, ', '.join(self._distributions))) def set_entry_point(self, entry_point): """Set the entry point of this PEX environment. diff --git a/pex/pex_info.py b/pex/pex_info.py index d4e88242e..4cce1eb9d 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -265,7 +265,8 @@ def update(self, other): def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) + pex_info_copy['distributions'] = self._distributions.copy() return json.dumps(pex_info_copy, **kwargs) def copy(self): - return PexInfo(info=self._pex_info.copy()) + return self.from_json(self.dump()) diff --git a/setup.py b/setup.py index b50c313e2..95efce434 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ packages = [ 'pex', 'pex.bin', + 'pex.commands', ], install_requires = [ SETUPTOOLS_REQUIREMENT, @@ -52,6 +53,9 @@ 'pytest', ], entry_points = { + 'distutils.commands': [ + 'bdist_pex = pex.commands.bdist_pex:bdist_pex', + ], 'console_scripts': [ 'pex = pex.bin.pex:main', ],