From 409a5ca753c6be3677097a7774eccc63f7d66e44 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 17:18:16 -0500 Subject: [PATCH 01/72] Removing some ignores that are not relevant to astropy_helpers --- .gitignore | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index ac86adf9..f7fa1b6c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,15 +12,11 @@ __pycache__ # Other generated files MANIFEST -astropy/version.py -astropy/cython_version.py -astropy/astropy.cfg -astropy/wcs/include/wcsconfig.h +astropy_helpers/version.py # Sphinx _build _generated -docs/api # Packages/installer info From f277169ddab22eb2edbe2c456c30fc8a8442eb0b Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 17:19:06 -0500 Subject: [PATCH 02/72] Adding the most recent ez_setup.py and the setuptools_boostrap script from Astropy --- ez_setup.py | 382 ++++++++++++++++++++++++++++++++++++++++ setuptools_bootstrap.py | 34 ++++ 2 files changed, 416 insertions(+) create mode 100644 ez_setup.py create mode 100644 setuptools_bootstrap.py diff --git a/ez_setup.py b/ez_setup.py new file mode 100644 index 00000000..9dc2c872 --- /dev/null +++ b/ez_setup.py @@ -0,0 +1,382 @@ +#!python +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import os +import shutil +import sys +import tempfile +import tarfile +import optparse +import subprocess +import platform + +from distutils import log + +try: + from site import USER_SITE +except ImportError: + USER_SITE = None + +DEFAULT_VERSION = "1.4.2" +DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" + +def _python_cmd(*args): + args = (sys.executable,) + args + return subprocess.call(args) == 0 + +def _check_call_py24(cmd, *args, **kwargs): + res = subprocess.call(cmd, *args, **kwargs) + class CalledProcessError(Exception): + pass + if not res == 0: + msg = "Command '%s' return non-zero exit status %d" % (cmd, res) + raise CalledProcessError(msg) +vars(subprocess).setdefault('check_call', _check_call_py24) + +def _install(tarball, install_args=()): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # installing + log.warn('Installing Setuptools') + if not _python_cmd('setup.py', 'install', *install_args): + log.warn('Something went wrong during the installation.') + log.warn('See the error message above.') + # exitcode will be 2 + return 2 + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + + +def _build_egg(egg, tarball, to_dir): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # building an egg + log.warn('Building a Setuptools egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + # returning the result + log.warn(egg) + if not os.path.exists(egg): + raise IOError('Could not build the egg.') + + +def _do_download(version, download_base, to_dir, download_delay): + egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' + % (version, sys.version_info[0], sys.version_info[1])) + if not os.path.exists(egg): + tarball = download_setuptools(version, download_base, + to_dir, download_delay) + _build_egg(egg, tarball, to_dir) + sys.path.insert(0, egg) + + # Remove previously-imported pkg_resources if present (see + # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). + if 'pkg_resources' in sys.modules: + del sys.modules['pkg_resources'] + + import setuptools + setuptools.bootstrap_install_from = egg + + +def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, download_delay=15): + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + was_imported = 'pkg_resources' in sys.modules or \ + 'setuptools' in sys.modules + try: + import pkg_resources + except ImportError: + return _do_download(version, download_base, to_dir, download_delay) + try: + pkg_resources.require("setuptools>=" + version) + return + except pkg_resources.VersionConflict: + e = sys.exc_info()[1] + if was_imported: + sys.stderr.write( + "The required version of setuptools (>=%s) is not available,\n" + "and can't be installed while this script is running. Please\n" + "install a more recent version first, using\n" + "'easy_install -U setuptools'." + "\n\n(Currently using %r)\n" % (version, e.args[0])) + sys.exit(2) + else: + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return _do_download(version, download_base, to_dir, + download_delay) + except pkg_resources.DistributionNotFound: + return _do_download(version, download_base, to_dir, + download_delay) + +def _clean_check(cmd, target): + """ + Run the command to download target. If the command fails, clean up before + re-raising the error. + """ + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + if os.access(target, os.F_OK): + os.unlink(target) + raise + +def download_file_powershell(url, target): + """ + Download the file at url to target using Powershell (which will validate + trust). Raise an exception if the command cannot complete. + """ + target = os.path.abspath(target) + cmd = [ + 'powershell', + '-Command', + "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), + ] + _clean_check(cmd, target) + +def has_powershell(): + if platform.system() != 'Windows': + return False + cmd = ['powershell', '-Command', 'echo test'] + devnull = open(os.path.devnull, 'wb') + try: + try: + subprocess.check_call(cmd, stdout=devnull, stderr=devnull) + except: + return False + finally: + devnull.close() + return True + +download_file_powershell.viable = has_powershell + +def download_file_curl(url, target): + cmd = ['curl', url, '--silent', '--output', target] + _clean_check(cmd, target) + +def has_curl(): + cmd = ['curl', '--version'] + devnull = open(os.path.devnull, 'wb') + try: + try: + subprocess.check_call(cmd, stdout=devnull, stderr=devnull) + except: + return False + finally: + devnull.close() + return True + +download_file_curl.viable = has_curl + +def download_file_wget(url, target): + cmd = ['wget', url, '--quiet', '--output-document', target] + _clean_check(cmd, target) + +def has_wget(): + cmd = ['wget', '--version'] + devnull = open(os.path.devnull, 'wb') + try: + try: + subprocess.check_call(cmd, stdout=devnull, stderr=devnull) + except: + return False + finally: + devnull.close() + return True + +download_file_wget.viable = has_wget + +def download_file_insecure(url, target): + """ + Use Python to download the file, even though it cannot authenticate the + connection. + """ + try: + from urllib.request import urlopen + except ImportError: + from urllib2 import urlopen + src = dst = None + try: + src = urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = src.read() + dst = open(target, "wb") + dst.write(data) + finally: + if src: + src.close() + if dst: + dst.close() + +download_file_insecure.viable = lambda: True + +def get_best_downloader(): + downloaders = [ + download_file_powershell, + download_file_curl, + download_file_wget, + download_file_insecure, + ] + + for dl in downloaders: + if dl.viable(): + return dl + +def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, delay=15, + downloader_factory=get_best_downloader): + """Download setuptools from a specified location and return its filename + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download + attempt. + + ``downloader_factory`` should be a function taking no arguments and + returning a function for downloading a URL to a target. + """ + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + tgz_name = "setuptools-%s.tar.gz" % version + url = download_base + tgz_name + saveto = os.path.join(to_dir, tgz_name) + if not os.path.exists(saveto): # Avoid repeated downloads + log.warn("Downloading %s", url) + downloader = downloader_factory() + downloader(url, saveto) + return os.path.realpath(saveto) + + +def _extractall(self, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + import copy + import operator + from tarfile import ExtractError + directories = [] + + if members is None: + members = self + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directories with a safe mode. + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 448 # decimal for oct 0700 + self.extract(tarinfo, path) + + # Reverse sort directories. + if sys.version_info < (2, 4): + def sorter(dir1, dir2): + return cmp(dir1.name, dir2.name) + directories.sort(sorter) + directories.reverse() + else: + directories.sort(key=operator.attrgetter('name'), reverse=True) + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + dirpath = os.path.join(path, tarinfo.name) + try: + self.chown(tarinfo, dirpath) + self.utime(tarinfo, dirpath) + self.chmod(tarinfo, dirpath) + except ExtractError: + e = sys.exc_info()[1] + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + +def _build_install_args(options): + """ + Build the arguments to 'python setup.py install' on the setuptools package + """ + install_args = [] + if options.user_install: + if sys.version_info < (2, 6): + log.warn("--user requires Python 2.6 or later") + raise SystemExit(1) + install_args.append('--user') + return install_args + +def _parse_args(): + """ + Parse the command line for options + """ + parser = optparse.OptionParser() + parser.add_option( + '--user', dest='user_install', action='store_true', default=False, + help='install in user site package (requires Python 2.6 or later)') + parser.add_option( + '--download-base', dest='download_base', metavar="URL", + default=DEFAULT_URL, + help='alternative URL from where to download the setuptools package') + parser.add_option( + '--insecure', dest='downloader_factory', action='store_const', + const=lambda: download_file_insecure, default=get_best_downloader, + help='Use internal, non-validating downloader' + ) + options, args = parser.parse_args() + # positional arguments are ignored + return options + +def main(version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + options = _parse_args() + tarball = download_setuptools(download_base=options.download_base, + downloader_factory=options.downloader_factory) + return _install(tarball, _build_install_args(options)) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setuptools_bootstrap.py b/setuptools_bootstrap.py new file mode 100644 index 00000000..174ee424 --- /dev/null +++ b/setuptools_bootstrap.py @@ -0,0 +1,34 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +""" +Pre-ez_setup bootstrap module to ensure that either distribute or setuptools >= +0.7 is used (over pre-distribute setuptools) if it is available on the path; +otherwise the latest setuptools will be downloaded and bootstrapped with +``ez_setup.py``. +""" + +import sys +import imp + +try: + import pkg_resources + _setuptools_req = pkg_resources.Requirement.parse('setuptools>=0.7') + # This may raise a DistributionNotFound in which case no version of + # setuptools or distribute is properly instlaled + _setuptools = pkg_resources.get_distribution('setuptools') + if _setuptools not in _setuptools_req: + # Older version of setuptools; check if we have distribute; again if + # this results in DistributionNotFound we want to give up + _distribute = pkg_resources.get_distribution('distribute') + if _setuptools != _distribute: + # It's possible on some pathological systems to have an old version + # of setuptools and distribute on sys.path simultaneously; make + # sure distribute is the one that's used + sys.path.insert(1, _distribute.location) + _distribute.activate() + imp.reload(pkg_resources) +except: + # There are several types of exceptions that can occur here; if all else + # fails bootstrap and use the bootstrapped version + from ez_setup import use_setuptools + use_setuptools() From 2a5033a0a15048dbce430c7e16bf522c808379d2 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 17:24:15 -0500 Subject: [PATCH 03/72] Start an astropy_helpers package and initialize it wieht the setup_helpers and version_helpers modules from astropy; these are still heavily reliant on Astropy to work --- astropy_helpers/__init__.py | 0 astropy_helpers/setup_helpers.py | 1620 ++++++++++++++++++++++++++++ astropy_helpers/version_helpers.py | 251 +++++ 3 files changed, 1871 insertions(+) create mode 100644 astropy_helpers/__init__.py create mode 100644 astropy_helpers/setup_helpers.py create mode 100644 astropy_helpers/version_helpers.py diff --git a/astropy_helpers/__init__.py b/astropy_helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py new file mode 100644 index 00000000..9f2f784d --- /dev/null +++ b/astropy_helpers/setup_helpers.py @@ -0,0 +1,1620 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This module contains a number of utilities for use during +setup/build/packaging that are useful to astropy as a whole. +""" + +from __future__ import absolute_import, print_function + +import collections +import errno +import imp +import inspect +import os +import re +import shlex +import shutil +import subprocess +import sys +import textwrap +import warnings + +from distutils import log, ccompiler, sysconfig +from distutils.cmd import DistutilsOptionError +from distutils.dist import Distribution +from distutils.errors import DistutilsError, DistutilsFileError +from distutils.core import Extension +from distutils.core import Command +from distutils.command.sdist import sdist as DistutilsSdist +from setuptools.command.build_ext import build_ext as SetuptoolsBuildExt +from setuptools.command.build_py import build_py as SetuptoolsBuildPy + +from setuptools.command.register import register as SetuptoolsRegister +from setuptools import find_packages + +from .tests.helper import astropy_test +from .utils import silence +from .utils.compat.misc import invalidate_caches +from .utils.misc import walk_skip_hidden +from .utils.exceptions import AstropyDeprecationWarning + + +try: + import Cython + HAVE_CYTHON = True +except ImportError: + HAVE_CYTHON = False + + +try: + import sphinx + from sphinx.setup_command import BuildDoc as SphinxBuildDoc + HAVE_SPHINX = True +except ImportError: + HAVE_SPHINX = False +except SyntaxError: # occurs if markupsafe is recent version, which doesn't support Python 3.2 + HAVE_SPHINX = False + + +PY3 = sys.version_info[0] >= 3 + + +# This adds a new keyword to the setup() function +Distribution.skip_2to3 = [] + + +_adjusted_compiler = False +def adjust_compiler(package): + """ + This function detects broken compilers and switches to another. If + the environment variable CC is explicitly set, or a compiler is + specified on the commandline, no override is performed -- the purpose + here is to only override a default compiler. + + The specific compilers with problems are: + + * The default compiler in XCode-4.2, llvm-gcc-4.2, + segfaults when compiling wcslib. + + The set of broken compilers can be updated by changing the + compiler_mapping variable. It is a list of 2-tuples where the + first in the pair is a regular expression matching the version + of the broken compiler, and the second is the compiler to change + to. + """ + + compiler_mapping = [ + (b'i686-apple-darwin[0-9]*-llvm-gcc-4.2', 'clang') + ] + + global _adjusted_compiler + if _adjusted_compiler: + return + + # Whatever the result of this function is, it only needs to be run once + _adjusted_compiler = True + + if 'CC' in os.environ: + + # Check that CC is not set to llvm-gcc-4.2 + c_compiler = os.environ['CC'] + + try: + version = get_compiler_version(c_compiler) + except OSError: + msg = textwrap.dedent( + """ + The C compiler set by the CC environment variable: + + {compiler:s} + + cannot be found or executed. + """.format(compiler=c_compiler)) + log.warn(msg) + sys.exit(1) + + for broken, fixed in compiler_mapping: + if re.match(broken, version): + msg = textwrap.dedent( + """Compiler specified by CC environment variable + ({compiler:s}:{version:s}) will fail to compile {pkg:s}. + Please set CC={fixed:s} and try again. + You can do this, for example, by running: + + CC={fixed:s} python setup.py + + where is the command you ran. + """.format(compiler=c_compiler, version=version, + pkg=package, fixed=fixed)) + log.warn(msg) + sys.exit(1) + + # If C compiler is set via CC, and isn't broken, we are good to go. We + # should definitely not try accessing the compiler specified by + # ``sysconfig.get_config_var('CC')`` lower down, because this may fail + # if the compiler used to compile Python is missing (and maybe this is + # why the user is setting CC). For example, the official Python 2.7.3 + # MacOS X binary was compled with gcc-4.2, which is no longer available + # in XCode 4. + return + + if get_distutils_build_option('compiler'): + return + + compiler_type = ccompiler.get_default_compiler() + + if compiler_type == 'unix': + + # We have to get the compiler this way, as this is the one that is + # used if os.environ['CC'] is not set. It is actually read in from + # the Python Makefile. Note that this is not necessarily the same + # compiler as returned by ccompiler.new_compiler() + c_compiler = sysconfig.get_config_var('CC') + + try: + version = get_compiler_version(c_compiler) + except OSError: + msg = textwrap.dedent( + """ + The C compiler used to compile Python {compiler:s}, and + which is normally used to compile C extensions, is not + available. You can explicitly specify which compiler to + use by setting the CC environment variable, for example: + + CC=gcc python setup.py + + or if you are using MacOS X, you can try: + + CC=clang python setup.py + """.format(compiler=c_compiler)) + log.warn(msg) + sys.exit(1) + + + for broken, fixed in compiler_mapping: + if re.match(broken, version): + os.environ['CC'] = fixed + break + + +def get_compiler_version(compiler): + + process = subprocess.Popen( + shlex.split(compiler) + ['--version'], stdout=subprocess.PIPE) + + output = process.communicate()[0].strip() + try: + version = output.split()[0] + except IndexError: + return 'unknown' + + return version + + +def get_dummy_distribution(): + """Returns a distutils Distribution object used to instrument the setup + environment before calling the actual setup() function. + """ + + global _registered_commands + + if _registered_commands is None: + raise RuntimeError('astropy.setup_helpers.register_commands() must be ' + 'called before using ' + 'astropy.setup_helpers.get_dummy_distribution()') + + # Pre-parse the Distutils command-line options and config files to if + # the option is set. + dist = Distribution({'script_name': os.path.basename(sys.argv[0]), + 'script_args': sys.argv[1:]}) + dist.cmdclass.update(_registered_commands) + + with silence(): + try: + dist.parse_config_files() + dist.parse_command_line() + except (DistutilsError, AttributeError, SystemExit): + # Let distutils handle DistutilsErrors itself AttributeErrors can + # get raise for ./setup.py --help SystemExit can be raised if a + # display option was used, for example + pass + + return dist + + +def get_distutils_option(option, commands): + """ Returns the value of the given distutils option. + + Parameters + ---------- + option : str + The name of the option + + commands : list of str + The list of commands on which this option is available + + Returns + ------- + val : str or None + the value of the given distutils option. If the option is not set, + returns None. + """ + + dist = get_dummy_distribution() + + for cmd in commands: + cmd_opts = dist.command_options.get(cmd) + if cmd_opts is not None and option in cmd_opts: + return cmd_opts[option][1] + else: + return None + + +def get_distutils_build_option(option): + """ Returns the value of the given distutils build option. + + Parameters + ---------- + option : str + The name of the option + + Returns + ------- + val : str or None + The value of the given distutils build option. If the option + is not set, returns None. + """ + return get_distutils_option(option, ['build', 'build_ext', 'build_clib']) + + +def get_distutils_install_option(option): + """ Returns the value of the given distutils install option. + + Parameters + ---------- + option : str + The name of the option + + Returns + ------- + val : str or None + The value of the given distutils build option. If the option + is not set, returns None. + """ + return get_distutils_option(option, ['install']) + + +def get_distutils_build_or_install_option(option): + """ Returns the value of the given distutils build or install option. + + Parameters + ---------- + option : str + The name of the option + + Returns + ------- + val : str or None + The value of the given distutils build or install option. If the + option is not set, returns None. + """ + return get_distutils_option(option, ['build', 'build_ext', 'build_clib', + 'install']) + + +def get_compiler_option(): + """ Determines the compiler that will be used to build extension modules. + + Returns + ------- + compiler : str + The compiler option specificied for the build, build_ext, or build_clib + command; or the default compiler for the platform if none was + specified. + + """ + + compiler = get_distutils_build_option('compiler') + if compiler is None: + return ccompiler.get_default_compiler() + + return compiler + + +def get_debug_option(): + """ Determines if the build is in debug mode. + + Returns + ------- + debug : bool + True if the current build was started with the debug option, False + otherwise. + + """ + + try: + from .version import debug as current_debug + except ImportError: + current_debug = None + + # Only modify the debug flag if one of the build commands was explicitly + # run (i.e. not as a sub-command of something else) + dist = get_dummy_distribution() + if any(cmd in dist.commands for cmd in ['build', 'build_ext']): + debug = bool(get_distutils_build_option('debug')) + else: + debug = bool(current_debug) + + if current_debug is not None and current_debug != debug: + build_ext_cmd = dist.get_command_class('build_ext') + build_ext_cmd.force_rebuild = True + + return debug + + +_registered_commands = None +def register_commands(package, version, release): + global _registered_commands + + if _registered_commands is not None: + return _registered_commands + + _registered_commands = { + 'test': generate_test_command(package), + + # Use distutils' sdist because it respects package_data. + # setuptools/distributes sdist requires duplication of information in + # MANIFEST.in + 'sdist': DistutilsSdist, + + # The exact form of the build_ext command depends on whether or not + # we're building a release version + 'build_ext': generate_build_ext_command(package, release), + + # We have a custom build_py to generate the default configuration file + 'build_py': AstropyBuildPy, + + 'register': AstropyRegister + } + + try: + import bdist_mpkg + except ImportError: + pass + else: + # Use a custom command to build a dmg (on MacOS X) + _registered_commands['bdist_dmg'] = bdist_dmg + + + if HAVE_SPHINX: + _registered_commands['build_sphinx'] = AstropyBuildSphinx + else: + _registered_commands['build_sphinx'] = FakeBuildSphinx + + # Need to override the __name__ here so that the commandline options are + # presented as being related to the "build" command, for example; normally + # this wouldn't be necessary since commands also have a command_name + # attribute, but there is a bug in distutils' help display code that it + # uses __name__ instead of command_name. Yay distutils! + for name, cls in _registered_commands.items(): + cls.__name__ = name + + # Add a few custom options; more of these can be added by specific packages + # later + for option in [ + ('use-system-libraries', + "Use system libraries whenever possible", True)]: + add_command_option('build', *option) + add_command_option('install', *option) + + return _registered_commands + + +def generate_test_command(package_name): + return type(package_name + '_test_command', (astropy_test,), + {'package_name': package_name}) + + +def generate_build_ext_command(packagename, release): + """ + Creates a custom 'build_ext' command that allows for manipulating some of + the C extension options at build time. We use a function to build the + class since the base class for build_ext may be different depending on + certain build-time parameters (for example, we may use Cython's build_ext + instead of the default version in distutils). + + Uses the default distutils.command.build_ext by default. + """ + + uses_cython = should_build_with_cython(packagename, release) + + if uses_cython: + from Cython.Distutils import build_ext as basecls + else: + basecls = SetuptoolsBuildExt + + attrs = dict(basecls.__dict__) + orig_run = getattr(basecls, 'run', None) + orig_finalize = getattr(basecls, 'finalize_options', None) + + def finalize_options(self): + if orig_finalize is not None: + orig_finalize(self) + + # Generate + if self.uses_cython: + try: + from Cython import __version__ as cython_version + except ImportError: + # This shouldn't happen if we made it this far + cython_version = None + + if (cython_version is not None and + cython_version != self.uses_cython): + self.force_rebuild = True + # Update the used cython version + self.uses_cython = cython_version + + # Regardless of the value of the '--force' option, force a rebuild if + # the debug flag changed from the last build + if self.force_rebuild: + self.force = True + + def run(self): + # For extensions that require 'numpy' in their include dirs, replace + # 'numpy' with the actual paths + np_include = get_numpy_include_path() + for extension in self.extensions: + if 'numpy' in extension.include_dirs: + idx = extension.include_dirs.index('numpy') + extension.include_dirs.insert(idx, np_include) + extension.include_dirs.remove('numpy') + + # Replace .pyx with C-equivalents, unless c files are missing + for jdx, src in enumerate(extension.sources): + if src.endswith('.pyx'): + pyxfn = src + cfn = src[:-4] + '.c' + elif src.endswith('.c'): + pyxfn = src[:-2] + '.pyx' + cfn = src + + if os.path.isfile(pyxfn): + if self.uses_cython: + extension.sources[jdx] = pyxfn + else: + if os.path.isfile(cfn): + extension.sources[jdx] = cfn + else: + msg = ( + 'Could not find C file {0} for Cython file ' + '{1} when building extension {2}. ' + 'Cython must be installed to build from a ' + 'git checkout'.format(cfn, pyxfn, + extension.name)) + raise IOError(errno.ENOENT, msg, cfn) + + if orig_run is not None: + # This should always be the case for a correctly implemented + # distutils command. + orig_run(self) + + # Update cython_version.py if building with Cython + try: + from .version import cython_version + except ImportError: + cython_version = 'unknown' + if self.uses_cython and self.uses_cython != cython_version: + package_dir = os.path.relpath(packagename) + cython_py = os.path.join(package_dir, 'cython_version.py') + with open(cython_py, 'w') as f: + f.write('# Generated file; do not modify\n') + f.write('cython_version = {0!r}\n'.format(self.uses_cython)) + + if os.path.isdir(self.build_lib): + # The build/lib directory may not exist if the build_py command + # was not previously run, which may sometimes be the case + self.copy_file(cython_py, + os.path.join(self.build_lib, cython_py), + preserve_mode=False) + + invalidate_caches() + + if not self.distribution.is_pure() and os.path.isdir(self.build_lib): + # Finally, generate the default astropy.cfg; this can only be done + # after extension modules are built as some extension modules + # include config items. We only do this if it's not pure python, + # though, because if it is, we already did it in build_py + default_cfg = generate_default_config( + os.path.abspath(self.build_lib), + self.distribution.packages[0]) + if default_cfg: + default_cfg = os.path.relpath(default_cfg) + self.copy_file(default_cfg, + os.path.join(self.build_lib, default_cfg), + preserve_mode=False) + + attrs['run'] = run + attrs['finalize_options'] = finalize_options + attrs['force_rebuild'] = False + attrs['uses_cython'] = uses_cython + + return type('build_ext', (basecls, object), attrs) + + +class AstropyBuildPy(SetuptoolsBuildPy): + + def finalize_options(self): + # Update build_lib settings from the build command to always put + # build files in platform-specific subdirectories of build/, even + # for projects with only pure-Python source (this is desirable + # specifically for support of multiple Python version). + build_cmd = self.get_finalized_command('build') + plat_specifier = '.{0}-{1}'.format(build_cmd.plat_name, + sys.version[0:3]) + # Do this unconditionally + build_purelib = os.path.join(build_cmd.build_base, + 'lib' + plat_specifier) + build_cmd.build_purelib = build_purelib + build_cmd.build_lib = build_purelib + + # Ugly hack: We also need to 'fix' the build_lib option on the + # install command--it would be better just to override that command + # entirely, but we can get around that extra effort by doing it here + install_cmd = self.get_finalized_command('install') + install_cmd.build_lib = build_purelib + install_lib_cmd = self.get_finalized_command('install_lib') + install_lib_cmd.build_dir = build_purelib + self.build_lib = build_purelib + SetuptoolsBuildPy.finalize_options(self) + + def run_2to3(self, files, doctests=False): + # Filter the files to exclude things that shouldn't be 2to3'd + skip_2to3 = self.distribution.skip_2to3 + filtered_files = [] + for file in files: + for package in skip_2to3: + if file[len(self.build_lib) + 1:].startswith(package): + break + else: + filtered_files.append(file) + + SetuptoolsBuildPy.run_2to3(self, filtered_files, doctests) + + def run(self): + # first run the normal build_py + SetuptoolsBuildPy.run(self) + + if self.distribution.is_pure(): + # Generate the default astropy.cfg - we only do this here if it's + # pure python. Otherwise, it'll happen at the end of build_exp + default_cfg = generate_default_config( + os.path.abspath(self.build_lib), + self.distribution.packages[0]) + if default_cfg: + default_cfg = os.path.relpath(default_cfg) + self.copy_file(default_cfg, + os.path.join(self.build_lib, default_cfg), + preserve_mode=False) + + +def generate_default_config(build_lib, package): + config_path = os.path.relpath(package) + filename = os.path.join(config_path, package + '.cfg') + + if os.path.exists(filename): + log.info('regenerating default {0}.cfg file'.format(package)) + else: + log.info('generating default {0}.cfg file'.format(package)) + + if PY3: + builtins = 'builtins' + else: + builtins = '__builtin__' + + # astropy may have been built with a numpy that setuptools + # downloaded and installed into the current directory for us. + # Therefore, we need to extend the sys.path of the subprocess + # that's generating the config file, with the sys.path of this + # process. + + subproccode = ( + 'import sys; sys.path.extend({paths!r});' + 'import {builtins};{builtins}._ASTROPY_SETUP_ = True;' + 'from astropy.config.configuration import generate_all_config_items;' + 'generate_all_config_items({pkgnm!r}, True, filename={filenm!r})') + subproccode = subproccode.format(builtins=builtins, + pkgnm=package, + filenm=os.path.abspath(filename), + paths=sys.path) + + # Note that cwd=build_lib--we're importing astropy from the build/ dir + # but using the astropy/ source dir as the config directory + proc = subprocess.Popen([sys.executable, '-c', subproccode], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=build_lib) + stdout, stderr = proc.communicate() + + if proc.returncode == 0 and os.path.exists(filename): + return filename + else: + msg = ('Generation of default configuration item failed! Stdout ' + 'and stderr are shown below.\n' + 'Stdout:\n{stdout}\nStderr:\n{stderr}') + if isinstance(msg, bytes): + msg = msg.decode('UTF-8') + log.error(msg.format(stdout=stdout.decode('UTF-8'), + stderr=stderr.decode('UTF-8'))) + + +def add_command_option(command, name, doc, is_bool=False): + """ + Add a custom option to a setup command. + + Issues a warning if the option already exists on that command. + + Parameters + ---------- + command : str + The name of the command as given on the command line + + name : str + The name of the build option + + doc : str + A short description of the option, for the `--help` message + + is_bool : bool, optional + When `True`, the option is a boolean option and doesn't + require an associated value. + """ + + dist = get_dummy_distribution() + cmdcls = dist.get_command_class(command) + + attr = name.replace('-', '_') + + if hasattr(cmdcls, attr): + raise RuntimeError( + '{0!r} already has a {1!r} class attribute, barring {2!r} from ' + 'being usable as a custom option name.'.format(cmdcls, attr, name)) + + for idx, cmd in enumerate(cmdcls.user_options): + if cmd[0] == name: + log.warning('Overriding existing {0!r} option ' + '{1!r}'.format(command, name)) + del cmdcls.user_options[idx] + if name in cmdcls.boolean_options: + cmdcls.boolean_options.remove(name) + break + + cmdcls.user_options.append((name, None, doc)) + + if is_bool: + cmdcls.boolean_options.append(name) + + # Distutils' command parsing requires that a command object have an + # attribute with the same name as the option (with '-' replaced with '_') + # in order for that option to be recognized as valid + setattr(cmdcls, attr, None) + + +class AstropyRegister(SetuptoolsRegister): + """Extends the built in 'register' command to support a ``--hidden`` option + to make the registered version hidden on PyPI by default. + + The result of this is that when a version is registered as "hidden" it can + still be downloaded from PyPI, but it does not show up in the list of + actively supported versions under http://pypi.python.org/pypi/astropy, and + is not set as the most recent version. + + Although this can always be set through the web interface it may be more + convenient to be able to specify via the 'register' command. Hidden may + also be considered a safer default when running the 'register' command, + though this command uses distutils' normal behavior if the ``--hidden`` + option is omitted. + """ + + user_options = SetuptoolsRegister.user_options + [ + ('hidden', None, 'mark this release as hidden on PyPI by default') + ] + boolean_options = SetuptoolsRegister.boolean_options + ['hidden'] + + def initialize_options(self): + SetuptoolsRegister.initialize_options(self) + self.hidden = False + + def build_post_data(self, action): + data = SetuptoolsRegister.build_post_data(self, action) + if action == 'submit' and self.hidden: + data['_pypi_hidden'] = '1' + return data + + def _set_config(self): + # The original register command is buggy--if you use .pypirc with a + # server-login section *at all* the repository you specify with the -r + # option will be overwritten with either the repository in .pypirc or + # with the default, + # If you do not have a .pypirc using the -r option will just crash. + # Way to go distutils + + # If we don't set self.repository back to a default value _set_config + # can crash if there was a user-supplied value for this option; don't + # worry, we'll get the real value back afterwards + self.repository = 'pypi' + SetuptoolsRegister._set_config(self) + options = self.distribution.get_option_dict('register') + if 'repository' in options: + source, value = options['repository'] + # Really anything that came from setup.cfg or the command line + # should override whatever was in .pypirc + self.repository = value + + +if HAVE_SPHINX: + class AstropyBuildSphinx(SphinxBuildDoc): + """ A version of the ``build_sphinx`` command that uses the + version of Astropy that is built by the setup ``build`` command, + rather than whatever is installed on the system - to build docs + against the installed version, run ``make html`` in the + ``astropy/docs`` directory. + + This also automatically creates the docs/_static directories - + this is needed because github won't create the _static dir + because it has no tracked files. + """ + + description = 'Build Sphinx documentation for Astropy environment' + user_options = SphinxBuildDoc.user_options[:] + user_options.append(('warnings-returncode', 'w', + 'Parses the sphinx output and sets the return ' + 'code to 1 if there are any warnings. Note that ' + 'this will cause the sphinx log to only update ' + 'when it completes, rather than continuously as ' + 'is normally the case.')) + user_options.append(('clean-docs', 'l', + 'Completely clean previous builds, including ' + 'automodapi-generated files before building new ' + 'ones')) + user_options.append(('no-intersphinx', 'n', + 'Skip intersphinx, even if conf.py says to use ' + 'it')) + user_options.append(('open-docs-in-browser', 'o', + 'Open the docs in a browser (using the ' + 'webbrowser module) if the build finishes ' + 'successfully.')) + + boolean_options = SphinxBuildDoc.boolean_options[:] + boolean_options.append('warnings-returncode') + boolean_options.append('clean-docs') + boolean_options.append('no-intersphinx') + boolean_options.append('open-docs-in-browser') + + _self_iden_rex = re.compile(r"self\.([^\d\W][\w]+)", re.UNICODE) + + def initialize_options(self): + SphinxBuildDoc.initialize_options(self) + self.clean_docs = False + self.no_intersphinx = False + self.open_docs_in_browser = False + self.warnings_returncode = False + + def finalize_options(self): + #Clear out previous sphinx builds, if requested + if self.clean_docs: + dirstorm = ['docs/api'] + if self.build_dir is None: + dirstorm.append('docs/_build') + else: + dirstorm.append(self.build_dir) + + for d in dirstorm: + if os.path.isdir(d): + log.info('Cleaning directory ' + d) + shutil.rmtree(d) + else: + log.info('Not cleaning directory ' + d + ' because ' + 'not present or not a directory') + + SphinxBuildDoc.finalize_options(self) + + def run(self): + import webbrowser + + if PY3: + from urllib.request import pathname2url + else: + from urllib import pathname2url + + # This is used at the very end of `run` to decide if sys.exit should + # be called. If it's None, it won't be. + retcode = None + + # If possible, create the _static dir + if self.build_dir is not None: + # the _static dir should be in the same place as the _build dir + # for Astropy + basedir, subdir = os.path.split(self.build_dir) + if subdir == '': # the path has a trailing /... + basedir, subdir = os.path.split(basedir) + staticdir = os.path.join(basedir, '_static') + if os.path.isfile(staticdir): + raise DistutilsOptionError( + 'Attempted to build_sphinx in a location where' + + staticdir + 'is a file. Must be a directory.') + self.mkpath(staticdir) + + #Now make sure Astropy is built and determine where it was built + build_cmd = self.reinitialize_command('build') + build_cmd.inplace = 0 + self.run_command('build') + build_cmd = self.get_finalized_command('build') + build_cmd_path = os.path.abspath(build_cmd.build_lib) + + #Now generate the source for and spawn a new process that runs the + #command. This is needed to get the correct imports for the built + #version + + runlines, runlineno = inspect.getsourcelines(SphinxBuildDoc.run) + subproccode = textwrap.dedent(""" + from sphinx.setup_command import * + + os.chdir({srcdir!r}) + sys.path.insert(0, {build_cmd_path!r}) + + """).format(build_cmd_path=build_cmd_path, srcdir=self.source_dir) + #runlines[1:] removes 'def run(self)' on the first line + subproccode += textwrap.dedent(''.join(runlines[1:])) + + # All "self.foo" in the subprocess code needs to be replaced by the + # values taken from the current self in *this* process + subproccode = AstropyBuildSphinx._self_iden_rex.split(subproccode) + for i in range(1, len(subproccode), 2): + iden = subproccode[i] + val = getattr(self, iden) + if iden.endswith('_dir'): + #Directories should be absolute, because the `chdir` call + #in the new process moves to a different directory + subproccode[i] = repr(os.path.abspath(val)) + else: + subproccode[i] = repr(val) + subproccode = ''.join(subproccode) + + if self.no_intersphinx: + #the confoverrides variable in sphinx.setup_command.BuildDoc can + #be used to override the conf.py ... but this could well break + #if future versions of sphinx change the internals of BuildDoc, + #so remain vigilant! + subproccode = subproccode.replace('confoverrides = {}', + 'confoverrides = {\'intersphinx_mapping\':{}}') + + log.debug('Starting subprocess of {0} with python code:\n{1}\n' + '[CODE END])'.format(sys.executable, subproccode)) + + # To return the number of warnings, we need to capture stdout. This + # prevents a continuous updating at the terminal, but there's no + # apparent way around this. + if self.warnings_returncode: + proc = subprocess.Popen([sys.executable], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + stdo, stde = proc.communicate(subproccode) + + print(stdo) + + stdolines = stdo.split('\n') + + if stdolines[-2] == 'build succeeded.': + retcode = 0 + else: + retcode = 1 + + if retcode != 0: + if os.environ.get('TRAVIS', None) == 'true': + #this means we are in the travis build, so customize + #the message appropriately. + msg = ('The build_sphinx travis build FAILED ' + 'because sphinx issued documentation ' + 'warnings (scroll up to see the warnings).') + else: # standard failure message + msg = ('build_sphinx returning a non-zero exit ' + 'code because sphinx issued documentation ' + 'warnings.') + log.warn(msg) + + else: + proc = subprocess.Popen([sys.executable], stdin=subprocess.PIPE) + proc.communicate(subproccode.encode('utf-8')) + + if proc.returncode == 0: + if self.open_docs_in_browser: + if self.builder == 'html': + absdir = os.path.abspath(self.builder_target_dir) + index_path = os.path.join(absdir, 'index.html') + fileurl = 'file://' + pathname2url(index_path) + webbrowser.open(fileurl) + else: + log.warn('open-docs-in-browser option was given, but ' + 'the builder is not html! Ignogring.') + else: + log.warn('Sphinx Documentation subprocess failed with return ' + 'code ' + str(proc.returncode)) + + if retcode is not None: + # this is potentially dangerous in that there might be something + # after the call to `setup` in `setup.py`, and exiting here will + # prevent that from running. But there's no other apparent way + # to signal what the return code should be. + sys.exit(retcode) + + +def get_distutils_display_options(): + """ Returns a set of all the distutils display options in their long and + short forms. These are the setup.py arguments such as --name or --version + which print the project's metadata and then exit. + + Returns + ------- + opts : set + The long and short form display option arguments, including the - or -- + """ + + short_display_opts = set('-' + o[1] for o in Distribution.display_options + if o[1]) + long_display_opts = set('--' + o[0] for o in Distribution.display_options) + + # Include -h and --help which are not explicitly listed in + # Distribution.display_options (as they are handled by optparse) + short_display_opts.add('-h') + long_display_opts.add('--help') + + # This isn't the greatest approach to hardcode these commands. + # However, there doesn't seem to be a good way to determine + # whether build *will be* run as part of the command at this + # phase. + display_commands = set([ + 'clean', 'register', 'setopt', 'saveopts', 'egg_info', + 'alias']) + + return short_display_opts.union(long_display_opts.union(display_commands)) + + +def is_distutils_display_option(): + """ Returns True if sys.argv contains any of the distutils display options + such as --version or --name. + """ + + display_options = get_distutils_display_options() + return bool(set(sys.argv[1:]).intersection(display_options)) + + +def update_package_files(srcdir, extensions, package_data, packagenames, + package_dirs): + """ + This function is deprecated and maintained for backward compatibility + with affiliated packages. Affiliated packages should update their + setup.py to use `get_package_info` instead. + """ + warnings.warn( + "astropy.setup_helpers.update_package_files is deprecated. Update " + "your setup.py to use astropy.setup_helpers.get_package_info instead.", + AstropyDeprecationWarning) + + info = get_package_info(srcdir) + extensions.extend(info['ext_modules']) + package_data.update(info['package_data']) + packagenames = list(set(packagenames + info['packages'])) + package_dirs.update(info['package_dir']) + + +def get_package_info(srcdir): + """ + Collates all of the information for building all subpackages + subpackages and returns a dictionary of keyword arguments that can + be passed directly to `distutils.setup`. + + The purpose of this function is to allow subpackages to update the + arguments to the package's ``setup()`` function in its setup.py + script, rather than having to specify all extensions/package data + directly in the ``setup.py``. See Astropy's own + ``setup.py`` for example usage and the Astropy development docs + for more details. + + This function obtains that information by iterating through all + packages in ``srcdir`` and locating a ``setup_package.py`` module. + This module can contain the following functions: + ``get_extensions()``, ``get_package_data()``, + ``get_build_options()``, ``get_external_libraries()``, + and ``requires_2to3()``. + + Each of those functions take no arguments. + + - ``get_extensions`` returns a list of + `distutils.extension.Extension` objects. + + - ``get_package_data()`` returns a dict formatted as required by + the ``package_data`` argument to ``setup()``. + + - ``get_build_options()`` returns a list of tuples describing the + extra build options to add. + + - ``get_external_libraries()`` returns + a list of libraries that can optionally be built using external + dependencies. + + - ``requires_2to3()`` should return `True` when the source code + requires `2to3` processing to run on Python 3.x. If + ``requires_2to3()`` is missing, it is assumed to return `True`. + + """ + ext_modules = [] + packages = [] + package_data = {} + package_dir = {} + skip_2to3 = [] + + # Use the find_packages tool to locate all packages and modules + packages = filter_packages(find_packages()) + + # For each of the setup_package.py modules, extract any + # information that is needed to install them. The build options + # are extracted first, so that their values will be available in + # subsequent calls to `get_extensions`, etc. + for setuppkg in iter_setup_packages(srcdir): + if hasattr(setuppkg, 'get_build_options'): + options = setuppkg.get_build_options() + for option in options: + add_command_option('build', *option) + if hasattr(setuppkg, 'get_external_libraries'): + libraries = setuppkg.get_external_libraries() + for library in libraries: + add_external_library(library) + if hasattr(setuppkg, 'requires_2to3'): + requires_2to3 = setuppkg.requires_2to3() + else: + requires_2to3 = True + if not requires_2to3: + skip_2to3.append( + os.path.dirname(setuppkg.__file__)) + + for setuppkg in iter_setup_packages(srcdir): + # get_extensions must include any Cython extensions by their .pyx + # filename. + if hasattr(setuppkg, 'get_extensions'): + ext_modules.extend(setuppkg.get_extensions()) + if hasattr(setuppkg, 'get_package_data'): + package_data.update(setuppkg.get_package_data()) + + # Locate any .pyx files not already specified, and add their extensions in. + # The default include dirs include numpy to facilitate numerical work. + ext_modules.extend(get_cython_extensions(srcdir, ext_modules, ['numpy'])) + + # Now remove extensions that have the special name 'skip_cython', as they + # exist Only to indicate that the cython extensions shouldn't be built + for i, ext in reversed(list(enumerate(ext_modules))): + if ext.name == 'skip_cython': + del ext_modules[i] + + # On Microsoft compilers, we need to pass the '/MANIFEST' + # commandline argument. This was the default on MSVC 9.0, but is + # now required on MSVC 10.0, but it doesn't seeem to hurt to add + # it unconditionally. + if get_compiler_option() == 'msvc': + for ext in ext_modules: + ext.extra_link_args.append('/MANIFEST') + + return { + 'ext_modules': ext_modules, + 'packages': packages, + 'package_dir': package_dir, + 'package_data': package_data, + 'skip_2to3': skip_2to3 + } + + +def iter_setup_packages(srcdir): + """ A generator that finds and imports all of the ``setup_package.py`` + modules in the source packages. + + Returns + ------- + modgen : generator + A generator that yields (modname, mod), where `mod` is the module and + `modname` is the module name for the ``setup_package.py`` modules. + + """ + for root, dirs, files in walk_skip_hidden(srcdir): + if 'setup_package.py' in files: + filename = os.path.join(root, 'setup_package.py') + module = import_file(filename) + yield module + + +def iter_pyx_files(srcdir): + """ A generator that yields Cython source files (ending in '.pyx') in the + source packages. + + Returns + ------- + pyxgen : generator + A generator that yields (extmod, fullfn) where `extmod` is the + full name of the module that the .pyx file would live in based + on the source directory structure, and `fullfn` is the path to + the .pyx file. + + """ + for dirpath, dirnames, filenames in walk_skip_hidden(srcdir): + modbase = dirpath.replace(os.sep, '.') + for fn in filenames: + if fn.endswith('.pyx'): + fullfn = os.path.join(dirpath, fn) + # Package must match file name + extmod = modbase + '.' + fn[:-4] + yield (extmod, fullfn) + + +def should_build_with_cython(package, release=None): + """Returns the previously used Cython version (or 'unknown' if not + previously built) if Cython should be used to build extension modules from + pyx files. If the ``release`` parameter is not specified an attempt is + made to determine the release flag from `astropy.version`. + """ + + try: + version_module = __import__(package + '.cython_version', + fromlist=['release', 'cython_version']) + except ImportError: + version_module = None + + if release is None and version_module is not None: + try: + release = version_module.release + except AttributeError: + pass + + try: + cython_version = version_module.cython_version + except AttributeError: + cython_version = 'unknown' + + # Only build with Cython if, of course, Cython is installed, we're in a + # development version (i.e. not release) or the Cython-generated source + # files haven't been created yet (cython_version == 'unknown'). The latter + # case can happen even when release is True if checking out a release tag + # from the repository + if HAVE_CYTHON and (not release or cython_version == 'unknown'): + return cython_version + else: + return False + + +def get_cython_extensions(srcdir, prevextensions=tuple(), extincludedirs=None): + """ Looks for Cython files and generates Extensions if needed. + + Parameters + ---------- + srcdir : str + Path to the root of the source directory to search. + prevextensions : list of `~distutils.core.Extension` objects + The extensions that are already defined. Any .pyx files already here + will be ignored. + extincludedirs : list of str or None + Directories to include as the `include_dirs` argument to the generated + `~distutils.core.Extension` objects. + + Returns + ------- + exts : list of `~distutils.core.Extension` objects + The new extensions that are needed to compile all .pyx files (does not + include any already in `prevextensions`). + """ + + # Vanilla setuptools and old versions of distribute include Cython files + # as .c files in the sources, not .pyx, so we cannot simply look for + # existing .pyx sources in the previous sources, but we should also check + # for .c files with the same remaining filename. So we look for .pyx and + # .c files, and we strip the extension. + + prevsourcepaths = [] + for ext in prevextensions: + for s in ext.sources: + if s.endswith(('.pyx', '.c')): + prevsourcepaths.append(os.path.realpath(os.path.splitext(s)[0])) + + ext_modules = [] + for extmod, pyxfn in iter_pyx_files(srcdir): + if os.path.realpath(os.path.splitext(pyxfn)[0]) not in prevsourcepaths: + ext_modules.append(Extension(extmod, [pyxfn], + include_dirs=extincludedirs)) + + return ext_modules + + +def write_if_different(filename, data): + """ Write `data` to `filename`, if the content of the file is different. + + Parameters + ---------- + filename : str + The file name to be written to. + data : bytes + The data to be written to `filename`. + """ + assert isinstance(data, bytes) + + if os.path.exists(filename): + with open(filename, 'rb') as fd: + original_data = fd.read() + else: + original_data = None + + if original_data != data: + with open(filename, 'wb') as fd: + fd.write(data) + + +def get_numpy_include_path(): + """ + Gets the path to the numpy headers. + """ + # We need to go through this nonsense in case setuptools + # downloaded and installed Numpy for us as part of the build or + # install, since Numpy may still think it's in "setup mode", when + # in fact we're ready to use it to build astropy now. + + if sys.version_info[0] >= 3: + import builtins + if hasattr(builtins, '__NUMPY_SETUP__'): + del builtins.__NUMPY_SETUP__ + import imp + import numpy + imp.reload(numpy) + else: + import __builtin__ + if hasattr(__builtin__, '__NUMPY_SETUP__'): + del __builtin__.__NUMPY_SETUP__ + import numpy + reload(numpy) + + try: + numpy_include = numpy.get_include() + except AttributeError: + numpy_include = numpy.get_numpy_include() + return numpy_include + + +def import_file(filename): + """ + Imports a module from a single file as if it doesn't belong to a + particular package. + """ + # Specifying a traditional dot-separated fully qualified name here + # results in a number of "Parent module 'astropy' not found while + # handling absolute import" warnings. Using the same name, the + # namespaces of the modules get merged together. So, this + # generates an underscore-separated name which is more likely to + # be unique, and it doesn't really matter because the name isn't + # used directly here anyway. + with open(filename, 'U') as fd: + name = '_'.join( + os.path.relpath(os.path.splitext(filename)[0]).split(os.sep)[1:]) + return imp.load_module(name, fd, filename, ('.py', 'U', 1)) + + +class DistutilsExtensionArgs(collections.defaultdict): + """ + A special dictionary whose default values are the empty list. + + This is useful for building up a set of arguments for + `distutils.Extension` without worrying whether the entry is + already present. + """ + def __init__(self, *args, **kwargs): + def default_factory(): + return [] + + super(DistutilsExtensionArgs, self).__init__( + default_factory, *args, **kwargs) + + def update(self, other): + for key, val in other.items(): + self[key].extend(val) + + +def pkg_config(packages, default_libraries): + """ + Uses pkg-config to update a set of distutils Extension arguments + to include the flags necessary to link against the given packages. + + If the pkg-config lookup fails, default_libraries is applied to + libraries. + + Parameters + ---------- + packages : list of str + A list of pkg-config packages to look up. + + default_libraries : list of str + A list of library names to use if the pkg-config lookup fails. + + Returns + ------- + config : dict + A dictionary containing keyword arguments to + `distutils.Extension`. These entries include: + + - ``include_dirs``: A list of include directories + - ``library_dirs``: A list of library directories + - ``libraries``: A list of libraries + - ``define_macros``: A list of macro defines + - ``undef_macros``: A list of macros to undefine + - ``extra_compile_args``: A list of extra arguments to pass to + the compiler + """ + + flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries', + '-D': 'define_macros', '-U': 'undef_macros'} + command = "pkg-config --libs --cflags {0}".format(' '.join(packages)), + + result = DistutilsExtensionArgs() + + try: + pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + output = pipe.communicate()[0].strip() + except subprocess.CalledProcessError as e: + lines = [ + "pkg-config failed. This may cause the build to fail below.", + " command: {0}".format(e.cmd), + " returncode: {0}".format(e.returncode), + " output: {0}".format(e.output) + ] + log.warn('\n'.join(lines)) + result['libraries'].extend(default_libraries) + else: + if pipe.returncode != 0: + lines = [ + "pkg-config could not lookup up package(s) {0}.".format( + ", ".join(packages)), + "This may cause the build to fail below." + ] + log.warn('\n'.join(lines)) + result['libraries'].extend(default_libraries) + else: + for token in output.split(): + # It's not clear what encoding the output of + # pkg-config will come to us in. It will probably be + # some combination of pure ASCII (for the compiler + # flags) and the filesystem encoding (for any argument + # that includes directories or filenames), but this is + # just conjecture, as the pkg-config documentation + # doesn't seem to address it. + arg = token[:2].decode('ascii') + value = token[2:].decode(sys.getfilesystemencoding()) + if arg in flag_map: + if arg == '-D': + value = tuple(value.split('=', 1)) + result[flag_map[arg]].append(value) + else: + result['extra_compile_args'].append(value) + + return result + + +def add_external_library(library): + """ + Add a build option for selecting the internal or system copy of a library. + + Parameters + ---------- + library : str + The name of the library. If the library is `foo`, the build + option will be called `--use-system-foo`. + """ + + for command in ['build', 'build_ext', 'install']: + add_command_option(command, str('use-system-' + library), + 'Use the system {0} library'.format(library), + is_bool=True) + + +def use_system_library(library): + """ + Returns `True` if the build configuration indicates that the given + library should use the system copy of the library rather than the + internal one. + + For the given library `foo`, this will be `True` if + `--use-system-foo` or `--use-system-libraries` was provided at the + commandline or in `setup.cfg`. + + Parameters + ---------- + library : str + The name of the library + + Returns + ------- + use_system : bool + `True` if the build should use the system copy of the library. + """ + return ( + get_distutils_build_or_install_option('use_system_{0}'.format(library)) + or get_distutils_build_or_install_option('use_system_libraries')) + + +def filter_packages(packagenames): + """ + Removes some packages from the package list that shouldn't be + installed on the current version of Python. + """ + + if PY3: + exclude = '_py2' + else: + exclude = '_py3' + + return [x for x in packagenames if not x.endswith(exclude)] + + +class bdist_dmg(Command): + """ + The bdist_dmg command is used to produce the disk image containing the + installer, and with a custom background and icon placement. + """ + + user_options = [ + ('background=', 'b', "background image to use (should be 500x500px)"), + ('dist-dir=', 'd', "directory to put final built distributions in") + ] + description = "Create a Mac OS X disk image with the package installer" + + def initialize_options(self): + self.dist_dir = None + self.background = None + self.finalized = False + + def finalize_options(self): + self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) + self.finalized = True + + def run(self): + + pkg_dir = os.path.join(self.dist_dir, 'pkg') + + # Remove directory if it already exists + if os.path.exists(pkg_dir): + shutil.rmtree(pkg_dir) + + # First create the package installer with bdist_mpkg + mpkg = self.reinitialize_command('bdist_mpkg', reinit_subcommands=1) + mpkg.dist_dir = pkg_dir + mpkg.ensure_finalized() + mpkg.run() + + # Find the name of the pkg file. Since we removed the dist directory + # at the start of the script, our pkg should be the only file there. + files = os.listdir(pkg_dir) + if len(files) != 1: + raise DistutilsFileError( + "Expected a single file in the {pkg_dir} " + "directory".format(pkg_dir=pkg_dir)) + pkg_file = os.path.basename(files[0]) + pkg_name = os.path.splitext(pkg_file)[0] + + # Build the docs + docs = self.reinitialize_command('build_sphinx', reinit_subcommands=1) + docs.ensure_finalized() + docs.run() + + # Copy over the docs to the dist directory + shutil.copytree(os.path.join(docs.build_dir, 'html'), + os.path.join(pkg_dir, 'Documentation')) + + # Copy over the background to the disk image + if self.background is not None: + background_dir = os.path.join(pkg_dir, '.background') + os.mkdir(background_dir) + shutil.copy2(self.background, + os.path.join(background_dir, 'background.png')) + + # Start creating the volume + dmg_path = os.path.join(self.dist_dir, pkg_name + '.dmg') + dmg_path_tmp = os.path.join(self.dist_dir, pkg_name + '_tmp.dmg') + volume_name = pkg_name + + # Remove existing dmg files + if os.path.exists(dmg_path): + os.remove(dmg_path) + if os.path.exists(dmg_path_tmp): + os.remove(dmg_path_tmp) + + # Check if a volume is already mounted + volume_path = os.path.join('/', 'Volumes', volume_name) + if os.path.exists(volume_path): + raise DistutilsFileError( + "A volume named {volume_name} is already mounted - please " + "eject this and try again".format(volume_name=volume_name)) + + shell_script = """ + + # Create DMG file + hdiutil create -volname {volume_name} -srcdir {pkg_dir} -fs HFS+ -fsargs "-c c=64,a=16,e=16" -format UDRW -size 24m {dmg_path_tmp} + + # Mount disk image, and keep reference to device + device=$(hdiutil attach -readwrite -noverify -noautoopen {dmg_path_tmp} | egrep '^/dev/' | sed 1q | awk '{{print $1}}') + + echo ' + tell application "Finder" + tell disk "{volume_name}" + open + set current view of container window to icon view + set toolbar visible of container window to false + set statusbar visible of container window to false + set the bounds of container window to {{100, 100, 600, 600}} + set theViewOptions to the icon view options of container window + set arrangement of theViewOptions to not arranged + set icon size of theViewOptions to 128 + set the background picture of theViewOptions to file ".background:background.png" + set position of item "{pkg_file}" of container window to {{125, 320}} + set position of item "Documentation" of container window to {{375, 320}} + close + open + update without registering applications + delay 5 + end tell + end tell + ' | osascript + + # Eject disk image + hdiutil detach ${{device}} + + # Convert to final read-only disk image + hdiutil convert {dmg_path_tmp} -format UDZO -imagekey zlib-level=9 -o {dmg_path} + + """.format(volume_name=volume_name, pkg_dir=pkg_dir, + pkg_file=pkg_file, dmg_path_tmp=dmg_path_tmp, + dmg_path=dmg_path) + + # Make the disk image with the above shell script + os.system(shell_script) + + # Remove temporary disk image + os.remove(dmg_path_tmp) + + +class FakeBuildSphinx(Command): + """ + A dummy build_sphinx command that is called if Sphinx is not + installed and displays a relevant error message + """ + + #user options inherited from sphinx.setup_command.BuildDoc + user_options = [ + ('fresh-env', 'E', '' ), + ('all-files', 'a', ''), + ('source-dir=', 's', ''), + ('build-dir=', None, ''), + ('config-dir=', 'c', ''), + ('builder=', 'b', ''), + ('project=', None, ''), + ('version=', None, ''), + ('release=', None, ''), + ('today=', None, ''), + ('link-index', 'i', ''), + ] + + #user options appended in astropy.setup_helpers.AstropyBuildSphinx + user_options.append(('warnings-returncode', 'w','')) + user_options.append(('clean-docs', 'l', '')) + user_options.append(('no-intersphinx', 'n', '')) + user_options.append(('open-docs-in-browser', 'o','')) + + + + def initialize_options(self): + try: + raise RuntimeError("Sphinx must be installed for build_sphinx") + except: + log.error('error : Sphinx must be installed for build_sphinx') + sys.exit(1) diff --git a/astropy_helpers/version_helpers.py b/astropy_helpers/version_helpers.py new file mode 100644 index 00000000..92c1a986 --- /dev/null +++ b/astropy_helpers/version_helpers.py @@ -0,0 +1,251 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +""" +Utilities for generating the version string for Astropy (or an affiliated +package) and the version.py module, which contains version info for the +package. + +Within the generated astropy.version module, the `major`, `minor`, and `bugfix` +variables hold the respective parts of the version number (bugfix is '0' if +absent). The `release` variable is True if this is a release, and False if this +is a development version of astropy. For the actual version string, use:: + + from astropy.version import version + +or:: + + from astropy import __version__ + +""" + +from __future__ import division + +import datetime +import imp +import os +import subprocess +import sys + +from distutils import log +from warnings import warn + + +def _version_split(version): + """ + Split a version string into major, minor, and bugfix numbers (with bugfix + optional, defaulting to 0). + """ + + for prerel in ('.dev', 'a', 'b', 'rc'): + if prerel in version: + version = version.split(prerel)[0] + + versplit = version.split('.') + major = int(versplit[0]) + minor = int(versplit[1]) + bugfix = 0 if len(versplit) < 3 else int(versplit[2]) + return major, minor, bugfix + + +def update_git_devstr(version, path=None): + """ + Updates the git revision string if and only if the path is being imported + directly from a git working copy. This ensures that the revision number in + the version string is accurate. + """ + + try: + # Quick way to determine if we're in git or not - returns '' if not + devstr = get_git_devstr(sha=True, show_warning=False, path=path) + except OSError: + return version + + if not devstr: + # Probably not in git so just pass silently + return version + + if 'dev' in version: # update to the current git revision + version_base = version.split('.dev', 1)[0] + devstr = get_git_devstr(sha=False, show_warning=False, path=path) + + return version_base + '.dev' + devstr + else: + #otherwise it's already the true/release version + return version + + +def get_git_devstr(sha=False, show_warning=True, path=None): + """ + Determines the number of revisions in this repository. + + Parameters + ---------- + sha : bool + If True, the full SHA1 hash will be returned. Otherwise, the total + count of commits in the repository will be used as a "revision + number". + + show_warning : bool + If True, issue a warning if git returns an error code, otherwise errors + pass silently. + + path : str or None + If a string, specifies the directory to look in to find the git + repository. If None, the location of the file this function is in + is used to infer the git repository location. If given a filename it + uses the directory containing that file. + + Returns + ------- + devversion : str + Either a string with the revsion number (if `sha` is False), the + SHA1 hash of the current commit (if `sha` is True), or an empty string + if git version info could not be identified. + + """ + + from .utils import find_current_module + + if path is None: + try: + mod = find_current_module(1, finddiff=True) + path = os.path.abspath(mod.__file__) + except (ValueError, AttributeError): + path = __file__ + if not os.path.isdir(path): + path = os.path.abspath(os.path.split(path)[0]) + + if sha: + cmd = 'rev-parse' # Faster for getting just the hash of HEAD + else: + cmd = 'rev-list' + + try: + p = subprocess.Popen(['git', cmd, 'HEAD'], cwd=path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = p.communicate() + except OSError as e: + if show_warning: + warn('Error running git: ' + str(e)) + return '' + + if p.returncode == 128: + if show_warning: + warn('No git repository present! Using default dev version.') + return '' + elif p.returncode != 0: + if show_warning: + warn('Git failed while determining revision count: ' + stderr) + return '' + + if sha: + return stdout.decode('utf-8')[:40] + else: + nrev = stdout.decode('utf-8').count('\n') + return str(nrev) + + +# This is used by setup.py to create a new version.py - see that file for +# details. Note that the imports have to be absolute, since this is also used +# by affiliated packages. + +_FROZEN_VERSION_PY_TEMPLATE = """ +# Autogenerated by {packagename}'s setup.py on {timestamp} + +from astropy.version_helpers import update_git_devstr, get_git_devstr + +_last_generated_version = {verstr!r} + +version = update_git_devstr(_last_generated_version) +githash = get_git_devstr(sha=True, show_warning=False) + +major = {major} +minor = {minor} +bugfix = {bugfix} + +release = {rel} +debug = {debug} + +try: + from .utils._compiler import compiler +except ImportError: + compiler = "unknown" + +try: + from .cython_version import cython_version +except ImportError: + cython_version = "unknown" +"""[1:] + + +def _get_version_py_str(packagename, version, release, debug): + timestamp = str(datetime.datetime.now()) + major, minor, bugfix = _version_split(version) + if packagename.lower() == 'astropy': + packagename = 'Astropy' + else: + packagename = 'Astropy-affiliated package ' + packagename + return _FROZEN_VERSION_PY_TEMPLATE.format(packagename=packagename, + timestamp=timestamp, + verstr=version, + major=major, + minor=minor, + bugfix=bugfix, + rel=release, debug=debug) + + +def generate_version_py(packagename, version, release=None, debug=None): + """Regenerate the version.py module if necessary.""" + + from .setup_helpers import is_distutils_display_option + from .utils.compat.misc import invalidate_caches + + try: + version_module = __import__(packagename + '.version', + fromlist=['_last_generated_version', + 'version', 'release', 'debug']) + try: + last_generated_version = version_module._last_generated_version + except AttributeError: + # Older version.py with no _last_generated_version; this will + # ensure a new version.py is written + last_generated_version = None + current_release = version_module.release + current_debug = version_module.debug + except ImportError: + version_module = None + last_generated_version = None + current_release = None + current_debug = None + + if release is None: + # Keep whatever the current value is, if it exists + release = bool(current_release) + + if debug is None: + # Likewise, keep whatever the current value is, if it exists + debug = bool(current_debug) + + version_py = os.path.join(packagename, 'version.py') + + if (last_generated_version != version or current_release != release or + current_debug != debug): + if '-q' not in sys.argv and '--quiet' not in sys.argv: + log.set_threshold(log.INFO) + + if is_distutils_display_option(): + # Always silence unnecessary log messages when display options are + # being used + log.set_threshold(log.WARN) + + log.info('Freezing version number to {0}'.format(version_py)) + + with open(version_py, 'w') as f: + # This overwrites the actual version.py + f.write(_get_version_py_str(packagename, version, release, debug)) + + invalidate_caches() + + if version_module: + imp.reload(version_module) From 25cc5e44ff59f75a853248e22bb72692ec1095ad Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 17:35:38 -0500 Subject: [PATCH 04/72] Removed most requirements for astropy itself in the setup_helpers module, mainly by copying over a few utility functions (some of these utilities are no longer used in astropy as a result--should we remove them?) Also removed any support for the astropy test and configuration frameworks, but that can be added back in later as needed... --- astropy_helpers/setup_helpers.py | 66 +------------- astropy_helpers/utils.py | 145 +++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 65 deletions(-) create mode 100644 astropy_helpers/utils.py diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index 9f2f784d..fd623d50 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -32,11 +32,7 @@ from setuptools.command.register import register as SetuptoolsRegister from setuptools import find_packages -from .tests.helper import astropy_test -from .utils import silence -from .utils.compat.misc import invalidate_caches -from .utils.misc import walk_skip_hidden -from .utils.exceptions import AstropyDeprecationWarning +from .utils import silence, invalidate_caches, walk_skip_hidden try: @@ -360,8 +356,6 @@ def register_commands(package, version, release): return _registered_commands _registered_commands = { - 'test': generate_test_command(package), - # Use distutils' sdist because it respects package_data. # setuptools/distributes sdist requires duplication of information in # MANIFEST.in @@ -410,11 +404,6 @@ def register_commands(package, version, release): return _registered_commands -def generate_test_command(package_name): - return type(package_name + '_test_command', (astropy_test,), - {'package_name': package_name}) - - def generate_build_ext_command(packagename, release): """ Creates a custom 'build_ext' command that allows for manipulating some of @@ -598,55 +587,6 @@ def run(self): preserve_mode=False) -def generate_default_config(build_lib, package): - config_path = os.path.relpath(package) - filename = os.path.join(config_path, package + '.cfg') - - if os.path.exists(filename): - log.info('regenerating default {0}.cfg file'.format(package)) - else: - log.info('generating default {0}.cfg file'.format(package)) - - if PY3: - builtins = 'builtins' - else: - builtins = '__builtin__' - - # astropy may have been built with a numpy that setuptools - # downloaded and installed into the current directory for us. - # Therefore, we need to extend the sys.path of the subprocess - # that's generating the config file, with the sys.path of this - # process. - - subproccode = ( - 'import sys; sys.path.extend({paths!r});' - 'import {builtins};{builtins}._ASTROPY_SETUP_ = True;' - 'from astropy.config.configuration import generate_all_config_items;' - 'generate_all_config_items({pkgnm!r}, True, filename={filenm!r})') - subproccode = subproccode.format(builtins=builtins, - pkgnm=package, - filenm=os.path.abspath(filename), - paths=sys.path) - - # Note that cwd=build_lib--we're importing astropy from the build/ dir - # but using the astropy/ source dir as the config directory - proc = subprocess.Popen([sys.executable, '-c', subproccode], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=build_lib) - stdout, stderr = proc.communicate() - - if proc.returncode == 0 and os.path.exists(filename): - return filename - else: - msg = ('Generation of default configuration item failed! Stdout ' - 'and stderr are shown below.\n' - 'Stdout:\n{stdout}\nStderr:\n{stderr}') - if isinstance(msg, bytes): - msg = msg.decode('UTF-8') - log.error(msg.format(stdout=stdout.decode('UTF-8'), - stderr=stderr.decode('UTF-8'))) - - def add_command_option(command, name, doc, is_bool=False): """ Add a custom option to a setup command. @@ -996,10 +936,6 @@ def update_package_files(srcdir, extensions, package_data, packagenames, with affiliated packages. Affiliated packages should update their setup.py to use `get_package_info` instead. """ - warnings.warn( - "astropy.setup_helpers.update_package_files is deprecated. Update " - "your setup.py to use astropy.setup_helpers.get_package_info instead.", - AstropyDeprecationWarning) info = get_package_info(srcdir) extensions.extend(info['ext_modules']) diff --git a/astropy_helpers/utils.py b/astropy_helpers/utils.py new file mode 100644 index 00000000..882c2226 --- /dev/null +++ b/astropy_helpers/utils.py @@ -0,0 +1,145 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import contextlib +import sys + + +# Python 3.3's importlib caches filesystem reads for faster imports in the +# general case. But sometimes it's necessary to manually invalidate those +# caches so that the import system can pick up new generated files. See +# https://github.com/astropy/astropy/issues/820 +if sys.version_info[:2] >= (3, 3): + from importlib import invalidate_caches +else: + invalidate_caches = lambda: None + + +class _DummyFile(object): + """A noop writeable object.""" + + def write(self, s): + pass + + +@contextlib.contextmanager +def silence(): + """A context manager that silences sys.stdout and sys.stderr.""" + + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = _DummyFile() + sys.stderr = _DummyFile() + yield + sys.stdout = old_stdout + sys.stderr = old_stderr + + +if sys.platform == 'win32': + import ctypes + + def _has_hidden_attribute(filepath): + """ + Returns True if the given filepath has the hidden attribute on + MS-Windows. Based on a post here: + http://stackoverflow.com/questions/284115/cross-platform-hidden-file-detection + """ + if isinstance(filepath, bytes): + filepath = filepath.decode(sys.getfilesystemencoding()) + try: + attrs = ctypes.windll.kernel32.GetFileAttributesW(filepath) + assert attrs != -1 + result = bool(attrs & 2) + except (AttributeError, AssertionError): + result = False + return result +else: + def _has_hidden_attribute(filepath): + return False + + +def is_path_hidden(filepath): + """ + Determines if a given file or directory is hidden. + + Parameters + ---------- + filepath : str + The path to a file or directory + + Returns + ------- + hidden : bool + Returns `True` if the file is hidden + """ + + name = os.path.basename(os.path.abspath(filepath)) + if isinstance(name, bytes): + is_dotted = name.startswith(b'.') + else: + is_dotted = name.startswith('.') + return is_dotted or _has_hidden_attribute(filepath) + + +def walk_skip_hidden(top, onerror=None, followlinks=False): + """ + A wrapper for `os.walk` that skips hidden files and directories. + + This function does not have the parameter `topdown` from + `os.walk`: the directories must always be recursed top-down when + using this function. + + See also + -------- + os.walk : For a description of the parameters + """ + + for root, dirs, files in os.walk( + top, topdown=True, onerror=onerror, + followlinks=followlinks): + # These lists must be updated in-place so os.walk will skip + # hidden directories + dirs[:] = [d for d in dirs if not is_path_hidden(d)] + files[:] = [f for f in files if not is_path_hidden(f)] + yield root, dirs, files + + +def write_if_different(filename, data): + """Write `data` to `filename`, if the content of the file is different. + + Parameters + ---------- + filename : str + The file name to be written to. + data : bytes + The data to be written to `filename`. + """ + + assert isinstance(data, bytes) + + if os.path.exists(filename): + with open(filename, 'rb') as fd: + original_data = fd.read() + else: + original_data = None + + if original_data != data: + with open(filename, 'wb') as fd: + fd.write(data) + + +def import_file(filename): + """ + Imports a module from a single file as if it doesn't belong to a + particular package. + """ + # Specifying a traditional dot-separated fully qualified name here + # results in a number of "Parent module 'astropy' not found while + # handling absolute import" warnings. Using the same name, the + # namespaces of the modules get merged together. So, this + # generates an underscore-separated name which is more likely to + # be unique, and it doesn't really matter because the name isn't + # used directly here anyway. + with open(filename, 'U') as fd: + name = '_'.join( + os.path.relpath(os.path.splitext(filename)[0]).split(os.sep)[1:]) + return imp.load_module(name, fd, filename, ('.py', 'U', 1)) From c7c102898189ef5781aa5c933a848afb333110fe Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 18:09:32 -0500 Subject: [PATCH 05/72] Split the git utils from version_helpers out into their own git_helpers module. This enabled creating a version of generate_version_py that can work without having Astropy or any other support libraries installed, and that also provides support for projects that don't use git (though they lose revision numbers in their version strings until and unless we add support for other VCS (I do already have code to do this for SVN and HG should we wish to add it)). --- astropy_helpers/__init__.py | 4 + astropy_helpers/git_helpers.py | 112 ++++++++++++++++++++ astropy_helpers/version_helpers.py | 164 ++++++++--------------------- 3 files changed, 162 insertions(+), 118 deletions(-) create mode 100644 astropy_helpers/git_helpers.py diff --git a/astropy_helpers/__init__.py b/astropy_helpers/__init__.py index e69de29b..5a8b56b3 100644 --- a/astropy_helpers/__init__.py +++ b/astropy_helpers/__init__.py @@ -0,0 +1,4 @@ +try: + from .version import version as __version__ +except ImportError: + __version__ = '' diff --git a/astropy_helpers/git_helpers.py b/astropy_helpers/git_helpers.py new file mode 100644 index 00000000..c16b21c8 --- /dev/null +++ b/astropy_helpers/git_helpers.py @@ -0,0 +1,112 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +""" +Utilities for retrieving revision information from a project's git repository. +""" + +# Do not remove the following comment; it is used by +# astropy_helpers.version_helpers to determine the beginning of the code in +# this module + +# BEGIN + + +import os +import subprocess +import warnings + + +def update_git_devstr(version, path=None): + """ + Updates the git revision string if and only if the path is being imported + directly from a git working copy. This ensures that the revision number in + the version string is accurate. + """ + + try: + # Quick way to determine if we're in git or not - returns '' if not + devstr = get_git_devstr(sha=True, show_warning=False, path=path) + except OSError: + return version + + if not devstr: + # Probably not in git so just pass silently + return version + + if 'dev' in version: # update to the current git revision + version_base = version.split('.dev', 1)[0] + devstr = get_git_devstr(sha=False, show_warning=False, path=path) + + return version_base + '.dev' + devstr + else: + #otherwise it's already the true/release version + return version + + +def get_git_devstr(sha=False, show_warning=True, path=None): + """ + Determines the number of revisions in this repository. + + Parameters + ---------- + sha : bool + If True, the full SHA1 hash will be returned. Otherwise, the total + count of commits in the repository will be used as a "revision + number". + + show_warning : bool + If True, issue a warning if git returns an error code, otherwise errors + pass silently. + + path : str or None + If a string, specifies the directory to look in to find the git + repository. If None, the location of the file this function is in + is used to infer the git repository location. If given a filename it + uses the directory containing that file. + + Returns + ------- + devversion : str + Either a string with the revsion number (if `sha` is False), the + SHA1 hash of the current commit (if `sha` is True), or an empty string + if git version info could not be identified. + + """ + + if path is None: + path = __file__ + + if not os.path.isdir(path): + path = os.path.abspath(os.path.dirname(path)) + + if sha: + cmd = 'rev-parse' # Faster for getting just the hash of HEAD + else: + cmd = 'rev-list' + + try: + p = subprocess.Popen(['git', cmd, 'HEAD'], cwd=path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = p.communicate() + except OSError as e: + if show_warning: + warnings.warn('Error running git: ' + str(e)) + return '' + + if p.returncode == 128: + if show_warning: + warnings.warn('No git repository present! Using default dev ' + 'version.') + return '' + elif p.returncode != 0: + if show_warning: + warnings.warn('Git failed while determining revision ' + 'count: ' + stderr) + return '' + + if sha: + return stdout.decode('utf-8')[:40] + else: + nrev = stdout.decode('utf-8').count('\n') + return str(nrev) diff --git a/astropy_helpers/version_helpers.py b/astropy_helpers/version_helpers.py index 92c1a986..4b927b70 100644 --- a/astropy_helpers/version_helpers.py +++ b/astropy_helpers/version_helpers.py @@ -23,11 +23,15 @@ import datetime import imp import os -import subprocess +import pkgutil import sys from distutils import log -from warnings import warn + +from . import git_helpers +from .setup_helpers import is_distutils_display_option +from .utils import invalidate_caches + def _version_split(version): @@ -47,118 +51,13 @@ def _version_split(version): return major, minor, bugfix -def update_git_devstr(version, path=None): - """ - Updates the git revision string if and only if the path is being imported - directly from a git working copy. This ensures that the revision number in - the version string is accurate. - """ - - try: - # Quick way to determine if we're in git or not - returns '' if not - devstr = get_git_devstr(sha=True, show_warning=False, path=path) - except OSError: - return version - - if not devstr: - # Probably not in git so just pass silently - return version - - if 'dev' in version: # update to the current git revision - version_base = version.split('.dev', 1)[0] - devstr = get_git_devstr(sha=False, show_warning=False, path=path) - - return version_base + '.dev' + devstr - else: - #otherwise it's already the true/release version - return version - - -def get_git_devstr(sha=False, show_warning=True, path=None): - """ - Determines the number of revisions in this repository. - - Parameters - ---------- - sha : bool - If True, the full SHA1 hash will be returned. Otherwise, the total - count of commits in the repository will be used as a "revision - number". - - show_warning : bool - If True, issue a warning if git returns an error code, otherwise errors - pass silently. - - path : str or None - If a string, specifies the directory to look in to find the git - repository. If None, the location of the file this function is in - is used to infer the git repository location. If given a filename it - uses the directory containing that file. - - Returns - ------- - devversion : str - Either a string with the revsion number (if `sha` is False), the - SHA1 hash of the current commit (if `sha` is True), or an empty string - if git version info could not be identified. - - """ - - from .utils import find_current_module - - if path is None: - try: - mod = find_current_module(1, finddiff=True) - path = os.path.abspath(mod.__file__) - except (ValueError, AttributeError): - path = __file__ - if not os.path.isdir(path): - path = os.path.abspath(os.path.split(path)[0]) - - if sha: - cmd = 'rev-parse' # Faster for getting just the hash of HEAD - else: - cmd = 'rev-list' - - try: - p = subprocess.Popen(['git', cmd, 'HEAD'], cwd=path, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - stdout, stderr = p.communicate() - except OSError as e: - if show_warning: - warn('Error running git: ' + str(e)) - return '' - - if p.returncode == 128: - if show_warning: - warn('No git repository present! Using default dev version.') - return '' - elif p.returncode != 0: - if show_warning: - warn('Git failed while determining revision count: ' + stderr) - return '' - - if sha: - return stdout.decode('utf-8')[:40] - else: - nrev = stdout.decode('utf-8').count('\n') - return str(nrev) - - # This is used by setup.py to create a new version.py - see that file for # details. Note that the imports have to be absolute, since this is also used # by affiliated packages. - _FROZEN_VERSION_PY_TEMPLATE = """ # Autogenerated by {packagename}'s setup.py on {timestamp} -from astropy.version_helpers import update_git_devstr, get_git_devstr - -_last_generated_version = {verstr!r} - -version = update_git_devstr(_last_generated_version) -githash = get_git_devstr(sha=True, show_warning=False) +{header} major = {major} minor = {minor} @@ -179,28 +78,57 @@ def get_git_devstr(sha=False, show_warning=True, path=None): """[1:] -def _get_version_py_str(packagename, version, release, debug): +_FROZEN_VERSION_PY_WITH_GIT_HEADER = """ +{git_helpers} + +_last_generated_version = {verstr!r} + +version = update_git_devstr(_last_generated_version) +githash = get_git_devstr(sha=True, show_warning=False) +"""[1:] + + +def _get_version_py_str(packagename, version, release, debug, uses_git=True): timestamp = str(datetime.datetime.now()) major, minor, bugfix = _version_split(version) + if packagename.lower() == 'astropy': packagename = 'Astropy' else: packagename = 'Astropy-affiliated package ' + packagename + + if uses_git: + loader = pkgutil.get_loader(git_helpers) + source_lines = (loader.get_source() or '').splitlines() + if not source_lines: + log.warn('Cannot get source code for astropy_helpers.git_helpers; ' + 'git support disabled.') + return _get_version_py_str(packagename, version, release, debug, + uses_git=False) + idx = 0 + for idx, line in enumerate(source_lines): + if line.startswith('# BEGIN'): + break + git_helpers_py = '\n'.join(source_lines[idx + 1:]) + header = _FROZEN_VERSION_PY_WITH_GIT_HEADER.format( + git_helpers=git_helpers_py, + verstr=version) + else: + header = 'version = {0!r}'.format(version) + return _FROZEN_VERSION_PY_TEMPLATE.format(packagename=packagename, timestamp=timestamp, - verstr=version, + header=header, major=major, minor=minor, bugfix=bugfix, rel=release, debug=debug) -def generate_version_py(packagename, version, release=None, debug=None): +def generate_version_py(packagename, version, release=None, debug=None, + uses_git=True): """Regenerate the version.py module if necessary.""" - from .setup_helpers import is_distutils_display_option - from .utils.compat.misc import invalidate_caches - try: version_module = __import__(packagename + '.version', fromlist=['_last_generated_version', @@ -208,9 +136,8 @@ def generate_version_py(packagename, version, release=None, debug=None): try: last_generated_version = version_module._last_generated_version except AttributeError: - # Older version.py with no _last_generated_version; this will - # ensure a new version.py is written - last_generated_version = None + last_generated_version = version_module.version + current_release = version_module.release current_debug = version_module.debug except ImportError: @@ -243,7 +170,8 @@ def generate_version_py(packagename, version, release=None, debug=None): with open(version_py, 'w') as f: # This overwrites the actual version.py - f.write(_get_version_py_str(packagename, version, release, debug)) + f.write(_get_version_py_str(packagename, version, release, debug, + uses_git=uses_git)) invalidate_caches() From 6e937ea69364e5b02453510db49dcee0f7b6375f Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 6 Dec 2013 18:17:29 -0500 Subject: [PATCH 06/72] Add a stub of a setup.py and a MANIFEST.in --- MANIFEST.in | 9 +++++++++ setup.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 MANIFEST.in create mode 100755 setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..2c4987f4 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include README.rst +include CHANGES.rst +include LICENSE.rst + +include ez_setup.py +include setuptools_bootstrap.py + +exclude *.pyc *.o +prune build diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..68c33e60 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import setuptools_bootstrap +from setuptools import setup +from astropy_helpers.version_helpers import generate_version_py + +NAME = 'astropy_helpers' +VERSION = '0.4.dev' +RELEASE = 'dev' not in VERSION +DOWNLOAD_BASE_URL = 'http://pypi.python.org/packages/source/a/astropy_helpers' + +generate_version_py(NAME, VERSION, RELEASE, False) + +# Use the updated version including the git rev count +from astropy_helpers.version import version as VERSION + +setup( + name=NAME, + version=VERSION, + description='', + provides=NAME, + author='The Astropy Developers', + author_email='astropy.team@gmail.com', + license='BSD', + url='http://astropy.org', + long_description='', + download_url='{0}/astropy-{1}.tar.gz'.format(DOWNLOAD_BASE_URL, VERSION), + classifiers=[], + cmdclass={}, + zip_safe=False, + packages=[NAME] +) From 2b3222831aacb5377f2c7e4b0aa904264f34a6a9 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 9 Dec 2013 15:38:41 -0500 Subject: [PATCH 07/72] Restore missing imports --- astropy_helpers/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astropy_helpers/utils.py b/astropy_helpers/utils.py index 882c2226..c14ea293 100644 --- a/astropy_helpers/utils.py +++ b/astropy_helpers/utils.py @@ -1,6 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import contextlib +import imp +import os import sys From f4cfc39640194ea459229edaadd4f965fb78f2d7 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 9 Dec 2013 15:42:45 -0500 Subject: [PATCH 08/72] Updated this error string; though this function will likely be removed at some point. --- astropy_helpers/setup_helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index fd623d50..402d4e01 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -195,9 +195,10 @@ def get_dummy_distribution(): global _registered_commands if _registered_commands is None: - raise RuntimeError('astropy.setup_helpers.register_commands() must be ' - 'called before using ' - 'astropy.setup_helpers.get_dummy_distribution()') + raise RuntimeError( + 'astropy_helpers.setup_helpers.register_commands() must be ' + 'called before using ' + 'astropy_helpers.setup_helpers.get_dummy_distribution()') # Pre-parse the Distutils command-line options and config files to if # the option is set. From 53d77df6cc1a8309cb9b1b34ccf002609938df20 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 9 Dec 2013 15:58:17 -0500 Subject: [PATCH 09/72] Add a helpers function for getting info out of the package's .version module; previously this was returning the version module in astropy_helpers itself. This changes the get_debug_option function such that it requires the package name. Will probably refactor functions like that into a class that is already instatiated with basic info about the package being built, such as its name. --- astropy_helpers/setup_helpers.py | 37 ++++++++++++++++++++++++++---- astropy_helpers/version_helpers.py | 7 +++--- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index 402d4e01..b480bee0 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -318,7 +318,7 @@ def get_compiler_option(): return compiler -def get_debug_option(): +def get_debug_option(packagename): """ Determines if the build is in debug mode. Returns @@ -330,8 +330,9 @@ def get_debug_option(): """ try: - from .version import debug as current_debug - except ImportError: + current_debug = get_pkg_version_module(packagename, + fromlist=['debug']) + except (ImportError, AttributeError): current_debug = None # Only modify the debug flag if one of the build commands was explicitly @@ -349,6 +350,31 @@ def get_debug_option(): return debug +# TODO: Move this into astropy_helpers.version_helpers once the dependency of +# version_helpers on *this* module has been resolved (IOW, once these modules +# have been refactored to reduce their interdependency) +def get_pkg_version_module(packagename, fromlist=None): + """Returns the package's .version module generated by + `astropy_helpers.version_helpers.generate_version_py`. Raises an + ImportError if the version module is not found. + + If ``fromlist`` is an iterable, return a tuple of the members of the + version module corresponding to the member names given in ``fromlist``. + Raises an `AttributeError` if any of these module members are not found. + """ + + if not fromlist: + # Due to a historical quirk of Python's import implementation, + # __import__ will not return submodules of a package if 'fromlist' is + # empty. + # TODO: For Python 3.1 and up it may be preferable to use importlib + # instead of the __import__ builtin + return __import__(packagename + '.version', fromlist=['']) + else: + mod = __import__(packagename + '.version', fromlist=fromlist) + return tuple(getattr(mod, member) for member in fromlist) + + _registered_commands = None def register_commands(package, version, release): global _registered_commands @@ -491,8 +517,9 @@ def run(self): # Update cython_version.py if building with Cython try: - from .version import cython_version - except ImportError: + cython_version = get_pkg_version_module( + packagename, fromlist=['cython_version']) + except (AttributeError, ImportError): cython_version = 'unknown' if self.uses_cython and self.uses_cython != cython_version: package_dir = os.path.relpath(packagename) diff --git a/astropy_helpers/version_helpers.py b/astropy_helpers/version_helpers.py index 4b927b70..95051510 100644 --- a/astropy_helpers/version_helpers.py +++ b/astropy_helpers/version_helpers.py @@ -29,7 +29,7 @@ from distutils import log from . import git_helpers -from .setup_helpers import is_distutils_display_option +from .setup_helpers import is_distutils_display_option, get_pkg_version_module from .utils import invalidate_caches @@ -130,9 +130,8 @@ def generate_version_py(packagename, version, release=None, debug=None, """Regenerate the version.py module if necessary.""" try: - version_module = __import__(packagename + '.version', - fromlist=['_last_generated_version', - 'version', 'release', 'debug']) + version_module = get_pkg_version_module(packagename) + try: last_generated_version = version_module._last_generated_version except AttributeError: From e0fbe590e5d0dfda4c55a71f02e50cf292044379 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 9 Dec 2013 16:55:59 -0500 Subject: [PATCH 10/72] Removing the last vestiges of the astropy config system for now, as it won't work with astropy_helpers in its current state. This can, and should be revisited once we have a better idea of what we're going to do about configuration. --- astropy_helpers/setup_helpers.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index b480bee0..0f26e7b0 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -537,20 +537,6 @@ def run(self): invalidate_caches() - if not self.distribution.is_pure() and os.path.isdir(self.build_lib): - # Finally, generate the default astropy.cfg; this can only be done - # after extension modules are built as some extension modules - # include config items. We only do this if it's not pure python, - # though, because if it is, we already did it in build_py - default_cfg = generate_default_config( - os.path.abspath(self.build_lib), - self.distribution.packages[0]) - if default_cfg: - default_cfg = os.path.relpath(default_cfg) - self.copy_file(default_cfg, - os.path.join(self.build_lib, default_cfg), - preserve_mode=False) - attrs['run'] = run attrs['finalize_options'] = finalize_options attrs['force_rebuild'] = False @@ -602,18 +588,6 @@ def run(self): # first run the normal build_py SetuptoolsBuildPy.run(self) - if self.distribution.is_pure(): - # Generate the default astropy.cfg - we only do this here if it's - # pure python. Otherwise, it'll happen at the end of build_exp - default_cfg = generate_default_config( - os.path.abspath(self.build_lib), - self.distribution.packages[0]) - if default_cfg: - default_cfg = os.path.relpath(default_cfg) - self.copy_file(default_cfg, - os.path.join(self.build_lib, default_cfg), - preserve_mode=False) - def add_command_option(command, name, doc, is_bool=False): """ From 153a29ecf70b1bdafa34bb09ca1e620fbfa335b3 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 16 Dec 2013 13:12:18 -0500 Subject: [PATCH 11/72] Since setuptools and PyPI 'canonicalize' underscores to dashes might as well just used the dashed name explcitly --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 68c33e60..45595ab7 100755 --- a/setup.py +++ b/setup.py @@ -2,13 +2,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import setuptools_bootstrap +import pkg_resources from setuptools import setup from astropy_helpers.version_helpers import generate_version_py NAME = 'astropy_helpers' VERSION = '0.4.dev' RELEASE = 'dev' not in VERSION -DOWNLOAD_BASE_URL = 'http://pypi.python.org/packages/source/a/astropy_helpers' +DOWNLOAD_BASE_URL = 'http://pypi.python.org/packages/source/a/astropy-helpers' generate_version_py(NAME, VERSION, RELEASE, False) @@ -16,10 +17,9 @@ from astropy_helpers.version import version as VERSION setup( - name=NAME, + name=pkg_resources.safe_name(NAME), # astropy_helpers -> astropy-helpers version=VERSION, description='', - provides=NAME, author='The Astropy Developers', author_email='astropy.team@gmail.com', license='BSD', From b52cc50e265b2e2ad5d9d2df60cfd1d99f5290e2 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 16 Dec 2013 18:27:18 -0500 Subject: [PATCH 12/72] Two improvements to the silence context manager: If an exception occurs in the body of the with statement, restore sys.stdout/err so that the exception is displayed properly. Also add a dummy flush method to _DummyFile as there is some code in distutils that calls sys.stdout.flush --- astropy_helpers/utils.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/astropy_helpers/utils.py b/astropy_helpers/utils.py index c14ea293..04708086 100644 --- a/astropy_helpers/utils.py +++ b/astropy_helpers/utils.py @@ -22,6 +22,9 @@ class _DummyFile(object): def write(self, s): pass + def flush(self): + pass + @contextlib.contextmanager def silence(): @@ -31,9 +34,19 @@ def silence(): old_stderr = sys.stderr sys.stdout = _DummyFile() sys.stderr = _DummyFile() - yield - sys.stdout = old_stdout - sys.stderr = old_stderr + exception_occurred = False + try: + yield + except: + exception_occurred = True + # Go ahead and clean up so that exception handling can work normally + sys.stdout = old_stdout + sys.stderr = old_stderr + raise + + if not exception_occurred: + sys.stdout = old_stdout + sys.stderr = old_stderr if sys.platform == 'win32': From 5189fdb64bd1c0f28facb1a1c0a700a91bef9531 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 16 Dec 2013 18:30:54 -0500 Subject: [PATCH 13/72] Initial version of the ah_bootstrap.py script (name tentative). This works similarly to setuptool' ez_setup.py in how it tries several different routes to find the astropy_helpers module and get it on sys.path. The default is to just import it from a submodule called astropy_helpers--this is the default that will be used when running the astropy repo, for example. But there are many other options. I've tested out all the major use cases, but once I add a test suite to astropy_helpers it will be nice to have tests for the boostrap script as well. --- ah_bootstrap.py | 340 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 ah_bootstrap.py diff --git a/ah_bootstrap.py b/ah_bootstrap.py new file mode 100644 index 00000000..34bb84ac --- /dev/null +++ b/ah_bootstrap.py @@ -0,0 +1,340 @@ +import setuptools_bootstrap + +import contextlib +import errno +import os +import re +import subprocess as sp +import sys + +from distutils import log +from distutils.debug import DEBUG +from setuptools import Distribution + + +if sys.version[0] < 3: + _str_types = (str, unicode) +else: + _str_types = (str, bytes) + + +# TODO: Maybe enable checking for a specific version of astropy_helpers? + + +def use_astropy_helpers(path='astropy_helpers', download_if_needed=True, + index_url=None): + """ + Ensure that the `astropy_helpers` module is available and is importable. + This supports automatic submodule initialization if astropy_helpers is + included in a project as a git submodule, or will download it from PyPI if + necessary. + + Parameters + ---------- + + path : str or None, optional + A filesystem path relative to the root of the project's source code + that should be added to `sys.path` so that `astropy_helpers` can be + imported from that path. + + If the path is a git submodule it will automatically be initialzed + and/or updated. + + The path may also be to a ``.tar.gz`` archive of the astropy_helpers + source distribution. In this case the archive is automatically + unpacked and made temporarily available on `sys.path` as a ``.egg`` + archive. + + If `None` skip straight to downloading. + + download_if_needed : bool, optional + If the provided filesystem path is not found an attempt will be made to + download astropy_helpers from PyPI. It will then be made temporarily + available on `sys.path` as a ``.egg`` archive (using the + ``setup_requires`` feature of setuptools. + + index_url : str, optional + If provided, use a different URL for the Python package index than the + main PyPI server. + """ + + if not isinstance(path, _str_types): + if path is not None: + raise TypeError('path must be a string or None') + + if not download_if_needed: + log.debug('a path was not given and download from PyPI was not ' + 'allowed so this is effectively a no-op') + return + elif not os.path.exists(path): + # Even if the given path does not exist on the filesystem, if it *is* a + # submodule, `git submodule init` will create it + is_submodule = _check_submodule(path) + if is_submodule and _directory_import(path, download_if_needed, + is_submodule=is_submodule): + # Successfully imported from submodule + return + + if download_if_needed: + log.warn('The requested path {0!r} for importing astropy_helpers ' + 'does not exist. Attempting download ' + 'instead.'.format(path)) + else: + raise _AHBootstrapSystemExit( + 'Error: The requested path {0!r} for importing ' + 'astropy_helpers does not exist.'.format(path)) + elif os.path.isdir(path): + if _directory_import(path, download_if_needed): + return + elif os.path.isfile(path): + # Handle importing from a source archive; this also uses setup_requires + # but points easy_install directly to the source archive + try: + _do_download(find_links=[path]) + except Exception as e: + if download_if_needed: + log.warn('{0}\nWill attempt to download astropy_helpers from ' + 'PyPI instead.'.format(str(e))) + else: + raise _AHBootstrapSystemExit(e.args[0]) + else: + msg = ('{0!r} is not a valid file or directory (it could be a ' + 'symlink?)'.format(path)) + if download_if_needed: + log.warn(msg) + else: + raise _AHBootstrapSystemExit(msg) + + # If we made it this far, go ahead and attempt to download/activate + try: + _do_download(index_url=index_url) + except Exception as e: + if DEBUG: + raise + else: + raise _AHBootstrapSystemExit(e.args[0]) + + +def _do_download(find_links=None, index_url=None): + try: + if find_links: + allow_hosts = '' + index_url = None + else: + allow_hosts = None + # Annoyingly, setuptools will not handle other arguments to + # Distribution (such as options) before handling setup_requires, so it + # is not straightfoward to programmatically augment the arguments which + # are passed to easy_install + class _Distribution(Distribution): + def get_option_dict(self, command_name): + opts = Distribution.get_option_dict(self, command_name) + if command_name == 'easy_install': + if find_links is not None: + opts['find_links'] = ('setup script', find_links) + if index_url is not None: + opts['index_url'] = ('setup script', index_url) + if allow_hosts is not None: + opts['allow_hosts'] = ('setup script', allow_hosts) + return opts + + attrs = {'setup_requires': ['astropy-helpers']} + + if DEBUG: + dist = _Distribution(attrs=attrs) + else: + with _silence(): + dist = _Distribution(attrs=attrs) + except Exception as e: + msg = 'Error retrieving astropy helpers from {0}:\n{1}' + if find_links: + source = find_links[0] + elif index_url: + source = index_url + else: + source = 'PyPI' + + raise Exception(msg.format(source, str(e))) + + +def _directory_import(path, download_if_needed, is_submodule=None): + # Return True on success, False on failure but download is allowed, and + # otherwise raise SystemExit + # Check to see if the path is a git submodule + if is_submodule is None: + is_submodule = _check_submodule(path) + + log.info( + 'Attempting to import astropy_helpers from {0} {1!r}'.format( + 'submodule' if is_submodule else 'directory', path)) + sys.path.insert(0, path) + try: + __import__('astropy_helpers') + return True + except ImportError: + sys.path.remove(path) + + if download_if_needed: + log.warn( + 'Failed to import astropy_helpers from {0!r}; will ' + 'attempt to download it from PyPI instead.'.format(path)) + else: + raise _AHBoostrapSystemExit( + 'Failed to import astropy_helpers from {0!r}:\n' + '{1}'.format(path)) + # Otherwise, success! + + +def _check_submodule(path): + try: + p = sp.Popen(['git', 'submodule', 'status', '--', path], + stdout=sp.PIPE, stderr=sp.PIPE) + stdout, stderr = p.communicate() + except OSError as e: + if DEBUG: + raise + + if e.errno == errno.ENOENT: + # The git command simply wasn't found; this is most likely the + # case on user systems that don't have git and are simply + # trying to install the package from PyPI or a source + # distribution. Silently ignore this case and simply don't try + # to use submodules + return False + else: + raise _AHBoostrapSystemExit( + 'An unexpected error occurred when running the ' + '`git submodule status` command:\n{0}'.format(str(e))) + + + if p.returncode != 0 or stderr: + # Unfortunately the return code alone cannot be relied on, as + # earler versions of git returned 0 even if the requested submodule + # does not exist + log.debug('git submodule command failed ' + 'unexpectedly:\n{0}'.format(sterr)) + return False + else: + # The stdout should only contain one line--the status of the + # requested submodule + m = _git_submodule_status_re.match(stdout) + if m: + # Yes, the path *is* a git submodule + _update_submodule(m.group('submodule'), m.group('status')) + return True + else: + log.warn( + 'Unexected output from `git submodule status`:\n{0}\n' + 'Will attempt import from {1!r} regardless.'.format( + stdout, path)) + return False + + +def _update_submodule(submodule, status): + if status == ' ': + # The submodule is up to date; no action necessary + return + elif status == '-': + cmd = ['update', '--init'] + log.info('Initializing submodule {0!r}'.format(submodule)) + elif status == '+': + cmd = ['update'] + log.info('Updating submodule {0!r}'.format(submodule)) + elif status == 'U': + raise _AHBoostrapSystemExit( + 'Error: Submodule {0} contains unresolved merge conflicts. ' + 'Please complete or abandon any changes in the submodule so that ' + 'it is in a usable state, then try again.'.format(submodule)) + else: + log.warn('Unknown status {0!r} for git submodule {1!r}. Will ' + 'attempt to use the submodule as-is, but try to ensure ' + 'that the submodule is in a clean state and contains no ' + 'conflicts or errors.\n{2}'.format(status, submodule, + _err_help_msg)) + return + + err_msg = None + + try: + p = sp.Popen(['git', 'submodule'] + cmd + ['--', submodule], + stdout=sp.PIPE, stderr=sp.PIPE) + stdout, stderr = p.communicate() + except OSError as e: + err_msg = str(e) + else: + if p.returncode != 0 or stderr: + err_msg = stderr + + if err_msg: + log.warn('An unexpected error occurred updating the git submodule ' + '{0!r}:\n{1}\n{2}'.format(submodule, err_msg, _err_help_msg)) + + +class _DummyFile(object): + """A noop writeable object.""" + + def write(self, s): + pass + + def flush(self): + pass + + +@contextlib.contextmanager +def _silence(): + """A context manager that silences sys.stdout and sys.stderr.""" + + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = _DummyFile() + sys.stderr = _DummyFile() + exception_occurred = False + try: + yield + except: + exception_occurred = True + # Go ahead and clean up so that exception handling can work normally + sys.stdout = old_stdout + sys.stderr = old_stderr + raise + + if not exception_occurred: + sys.stdout = old_stdout + sys.stderr = old_stderr + + +_err_help_msg = """ +If the problem persists consider installing astropy_helpers manually using pip +(`pip install astropy_helpers`) or by manually downloading the source archive, +extracting it, and installing by running `python setup.py install` from the +root of the extracted source code. +""" + + +class _AHBootstrapSystemExit(SystemExit): + def __init__(self, *args): + if not args: + msg = 'An unknown problem occurred bootstrapping astropy_helpers.' + else: + msg = args[0] + + msg += '\n' + _err_help_msg + + super(_AHBootstrapSystemExit, self).__init__(msg, *args[1:]) + + +# Output of `git submodule status` is as follows: +# +# 1: Status indicator: '-' for submodule is uninitialized, '+' if submodule is +# initialized but is not at the commit currently indicated in .gitmodules (and +# thus needs to be updated), or 'U' if the submodule is in an unstable state +# (i.e. has merge conflicts) +# +# 2. SHA-1 hash of the current commit of the submodule (we don't really need +# this information but it's useful for checking that the output is correct) +# +# 3. The output of `git describe` for the submodule's current commit hash (this +# includes for example what branches the commit is on) but only if the +# submodule is initialized. We ignore this information for now +_git_submodule_status_re = re.compile( + b'^(?P[+-U ])(?P[0-9a-f]{40}) (?P\S+)( .*)?$') From 8b8e74bd8d478c918f2a4ee767423f18c8421997 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 18 Dec 2013 12:06:03 -0500 Subject: [PATCH 14/72] Should be version_info --- ah_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 34bb84ac..09c0de47 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -12,7 +12,7 @@ from setuptools import Distribution -if sys.version[0] < 3: +if sys.version_info[0] < 3: _str_types = (str, unicode) else: _str_types = (str, bytes) From 1a37056614b27b69fd709e7398729bd783c305d7 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 13 Jan 2014 17:49:31 -0500 Subject: [PATCH 15/72] Fix typo --- ah_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 09c0de47..83011d2a 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -212,7 +212,7 @@ def _check_submodule(path): # earler versions of git returned 0 even if the requested submodule # does not exist log.debug('git submodule command failed ' - 'unexpectedly:\n{0}'.format(sterr)) + 'unexpectedly:\n{0}'.format(stderr)) return False else: # The stdout should only contain one line--the status of the From 22c624b1f1c768f382fc1f30ef2106eb28ac13ae Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 13:40:37 -0400 Subject: [PATCH 16/72] Only output an error warning if git submodule had a non-zero exit status. It appears that in some versions of git even a successful run can output some text to stderr; see https://github.com/astropy/astropy/pull/1563#issuecomment-37675210 --- ah_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 83011d2a..aa3f61df 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -262,7 +262,7 @@ def _update_submodule(submodule, status): except OSError as e: err_msg = str(e) else: - if p.returncode != 0 or stderr: + if p.returncode != 0: err_msg = stderr if err_msg: From 1a75b262e4caed45754dd18a37ae3763566a6254 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 13:45:23 -0400 Subject: [PATCH 17/72] Combine setuptools_bootstrap into ah_bootstrap so as to reduce the number of needed 'bootstrap' scripts :) --- ah_bootstrap.py | 38 ++++++++++++++++++++++++++++++++------ setuptools_bootstrap.py | 34 ---------------------------------- 2 files changed, 32 insertions(+), 40 deletions(-) delete mode 100644 setuptools_bootstrap.py diff --git a/ah_bootstrap.py b/ah_bootstrap.py index aa3f61df..6eb18693 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -1,22 +1,48 @@ -import setuptools_bootstrap - import contextlib import errno +import imp import os import re import subprocess as sp import sys -from distutils import log -from distutils.debug import DEBUG -from setuptools import Distribution - if sys.version_info[0] < 3: _str_types = (str, unicode) else: _str_types = (str, bytes) +# Some pre-setuptools checks to ensure that either distribute or setuptools >= +# 0.7 is used (over pre-distribute setuptools) if it is available on the path; +# otherwise the latest setuptools will be downloaded and bootstrapped with +# ``ez_setup.py``. This used to be included in a separate file called +# setuptools_bootstrap.py; but it was combined into ah_bootstrap.py +try: + import pkg_resources + _setuptools_req = pkg_resources.Requirement.parse('setuptools>=0.7') + # This may raise a DistributionNotFound in which case no version of + # setuptools or distribute is properly instlaled + _setuptools = pkg_resources.get_distribution('setuptools') + if _setuptools not in _setuptools_req: + # Older version of setuptools; check if we have distribute; again if + # this results in DistributionNotFound we want to give up + _distribute = pkg_resources.get_distribution('distribute') + if _setuptools != _distribute: + # It's possible on some pathological systems to have an old version + # of setuptools and distribute on sys.path simultaneously; make + # sure distribute is the one that's used + sys.path.insert(1, _distribute.location) + _distribute.activate() + imp.reload(pkg_resources) +except: + # There are several types of exceptions that can occur here; if all else + # fails bootstrap and use the bootstrapped version + from ez_setup import use_setuptools + use_setuptools() + +from distutils import log +from distutils.debug import DEBUG +from setuptools import Distribution # TODO: Maybe enable checking for a specific version of astropy_helpers? diff --git a/setuptools_bootstrap.py b/setuptools_bootstrap.py deleted file mode 100644 index 174ee424..00000000 --- a/setuptools_bootstrap.py +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -""" -Pre-ez_setup bootstrap module to ensure that either distribute or setuptools >= -0.7 is used (over pre-distribute setuptools) if it is available on the path; -otherwise the latest setuptools will be downloaded and bootstrapped with -``ez_setup.py``. -""" - -import sys -import imp - -try: - import pkg_resources - _setuptools_req = pkg_resources.Requirement.parse('setuptools>=0.7') - # This may raise a DistributionNotFound in which case no version of - # setuptools or distribute is properly instlaled - _setuptools = pkg_resources.get_distribution('setuptools') - if _setuptools not in _setuptools_req: - # Older version of setuptools; check if we have distribute; again if - # this results in DistributionNotFound we want to give up - _distribute = pkg_resources.get_distribution('distribute') - if _setuptools != _distribute: - # It's possible on some pathological systems to have an old version - # of setuptools and distribute on sys.path simultaneously; make - # sure distribute is the one that's used - sys.path.insert(1, _distribute.location) - _distribute.activate() - imp.reload(pkg_resources) -except: - # There are several types of exceptions that can occur here; if all else - # fails bootstrap and use the bootstrapped version - from ez_setup import use_setuptools - use_setuptools() From c3f841b9e099f506a751b8a6971b15e7e3d6590a Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 17:50:28 -0400 Subject: [PATCH 18/72] Fix a bug in get_git_devstr--I think the old default functionality of using __file__ may have made sense when this was used primarily by Astropy itself, but in the context of an affiliated package it makes no sense. --- astropy_helpers/git_helpers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/astropy_helpers/git_helpers.py b/astropy_helpers/git_helpers.py index c16b21c8..2974d7d9 100644 --- a/astropy_helpers/git_helpers.py +++ b/astropy_helpers/git_helpers.py @@ -60,9 +60,8 @@ def get_git_devstr(sha=False, show_warning=True, path=None): path : str or None If a string, specifies the directory to look in to find the git - repository. If None, the location of the file this function is in - is used to infer the git repository location. If given a filename it - uses the directory containing that file. + repository. If `None`, the current working directory is used. + If given a filename it uses the directory containing that file. Returns ------- @@ -74,7 +73,7 @@ def get_git_devstr(sha=False, show_warning=True, path=None): """ if path is None: - path = __file__ + path = os.getcwd() if not os.path.isdir(path): path = os.path.abspath(os.path.dirname(path)) From a1469987dfecf321a8198e838789abb0f07740f4 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 14:45:31 -0400 Subject: [PATCH 19/72] added astropy_helpers branch of package-template as a submodule in astropy_helpers.tests--this will allow us to use the package template itself as our test project for testing astropy_helpers :) --- .gitmodules | 3 +++ astropy_helpers/tests/testpackage | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 astropy_helpers/tests/testpackage diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..93825913 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "astropy_helpers/tests/testpackage"] + path = astropy_helpers/tests/testpackage + url = git@github.com:embray/package-template.git diff --git a/astropy_helpers/tests/testpackage b/astropy_helpers/tests/testpackage new file mode 160000 index 00000000..857b7ae6 --- /dev/null +++ b/astropy_helpers/tests/testpackage @@ -0,0 +1 @@ +Subproject commit 857b7ae67b5bfa6c0da3328a735aeb24df15f01f From c1dc83f85b776f489f2b32fe8d44092206433403 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 16:30:01 -0400 Subject: [PATCH 20/72] Make astropy_helpers.tests into a proper subpackage, but exclude it from sdists to keep those small. Running the astropy_helpers tests should be of interest to developers only for now. Also make sure the testpackage isn't recursed into for py.test since we don't want to run *its* tests --- MANIFEST.in | 3 ++- astropy_helpers/tests/__init__.py | 0 setup.cfg | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 astropy_helpers/tests/__init__.py create mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 2c4987f4..b12416c5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,8 @@ include CHANGES.rst include LICENSE.rst include ez_setup.py -include setuptools_bootstrap.py +include ah_bootstrap.py exclude *.pyc *.o prune build +prune astropy_helpers/tests diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..6d5f24bf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = astropy_helpers/tests/testpackage From 6cc226cae0f2940714635fb6d587cf072660a7e1 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 17:52:56 -0400 Subject: [PATCH 21/72] Add our first test of astropy_helpers--this test actually helped catch a bug which was fixed in c3f841b9e099f506a751b8a6971b15e7e3d6590a --- astropy_helpers/tests/__init__.py | 51 +++++++++++++++++++++++ astropy_helpers/tests/test_git_helpers.py | 41 ++++++++++++++++++ astropy_helpers/tests/testpackage | 2 +- 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 astropy_helpers/tests/test_git_helpers.py diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index e69de29b..25fd2f51 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -0,0 +1,51 @@ +import os +import shutil +import subprocess as sp +import sys + +import pytest + +PACKAGE_DIR = os.path.dirname(__file__) + + +def run_cmd(cmd, args, path=None): + """ + Runs a shell command with the given argument list. Changes directory to + ``path`` if given, otherwise runs the command in the current directory. + + Returns a 3-tuple of (stdout, stderr, exit code) + """ + + if path is not None: + # Transparently support py.path objects + path = str(path) + + p = sp.Popen([cmd] + list(args), stdout=sp.PIPE, stderr=sp.PIPE, + cwd=path) + streams = tuple(s.decode('latin1').strip() for s in p.communicate()) + return streams + (p.returncode,) + + +@pytest.fixture +def testpackage(tmpdir): + """Create a copy of the testpackage repository (containing the package + template) in a tempdir and change directories to that temporary copy. + + Also ensures that any previous imports of the test package are unloaded + from `sys.modules`. + """ + + tmp_package = tmpdir.join('testpackage') + shutil.copytree(os.path.join(PACKAGE_DIR, 'testpackage'), + str(tmp_package)) + os.chdir(str(tmp_package)) + + if 'packagename' in sys.modules: + del sys.modules['packagename'] + + if '' in sys.path: + sys.path.remove('') + + sys.path.insert(0, '') + + return tmp_package diff --git a/astropy_helpers/tests/test_git_helpers.py b/astropy_helpers/tests/test_git_helpers.py new file mode 100644 index 00000000..e7d94963 --- /dev/null +++ b/astropy_helpers/tests/test_git_helpers.py @@ -0,0 +1,41 @@ +import imp +import os +import re + +from . import * + + +_DEV_VERSION_RE = re.compile(r'\d+\.\d+(?:\.\d+)?\.dev(\d+)') + + +def test_update_git_devstr(testpackage): + """Tests that the commit number in the package's version string updates + after git commits even without re-running setup.py. + """ + + stdout, stderr, ret = run_cmd('setup.py', ['--version']) + assert ret == 0 + version = stdout.strip() + + m = _DEV_VERSION_RE.match(version) + assert m + + revcount = int(m.group(1)) + + import packagename + assert packagename.__version__ == version + + # Make a silly git commit + with open('.test', 'w'): + pass + + run_cmd('git', ['add', '.test']) + run_cmd('git', ['commit', '-m', 'test']) + + import packagename.version + imp.reload(packagename.version) + imp.reload(packagename) + + m = _DEV_VERSION_RE.match(packagename.__version__) + assert m + assert int(m.group(1)) == revcount + 1 diff --git a/astropy_helpers/tests/testpackage b/astropy_helpers/tests/testpackage index 857b7ae6..d1d9cf7e 160000 --- a/astropy_helpers/tests/testpackage +++ b/astropy_helpers/tests/testpackage @@ -1 +1 @@ -Subproject commit 857b7ae67b5bfa6c0da3328a735aeb24df15f01f +Subproject commit d1d9cf7ee2104af5614d85372c391006d77e3afa From 21db0cbd4a864bd2958913fcc7669ff9dba6af86 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 18:27:40 -0400 Subject: [PATCH 22/72] Run setup.py from within the same python interpreter using setuptools' sandboxing--this way we can capture test coverage properly --- astropy_helpers/tests/test_git_helpers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/astropy_helpers/tests/test_git_helpers.py b/astropy_helpers/tests/test_git_helpers.py index e7d94963..0ab4c44a 100644 --- a/astropy_helpers/tests/test_git_helpers.py +++ b/astropy_helpers/tests/test_git_helpers.py @@ -2,19 +2,22 @@ import os import re +from setuptools.sandbox import run_setup + from . import * _DEV_VERSION_RE = re.compile(r'\d+\.\d+(?:\.\d+)?\.dev(\d+)') -def test_update_git_devstr(testpackage): +def test_update_git_devstr(testpackage, capsys): """Tests that the commit number in the package's version string updates after git commits even without re-running setup.py. """ - stdout, stderr, ret = run_cmd('setup.py', ['--version']) - assert ret == 0 + run_setup('setup.py', ['--version']) + + stdout, stderr = capsys.readouterr() version = stdout.strip() m = _DEV_VERSION_RE.match(version) From 122013d35c5355ffbe0dac9ad5f62b9495dbb31e Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 18:32:00 -0400 Subject: [PATCH 23/72] A further enhancement that enables coverage for most of astropy_helpers.git_helpers.update_git_devstr --- astropy_helpers/tests/test_git_helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/astropy_helpers/tests/test_git_helpers.py b/astropy_helpers/tests/test_git_helpers.py index 0ab4c44a..282d2157 100644 --- a/astropy_helpers/tests/test_git_helpers.py +++ b/astropy_helpers/tests/test_git_helpers.py @@ -42,3 +42,13 @@ def test_update_git_devstr(testpackage, capsys): m = _DEV_VERSION_RE.match(packagename.__version__) assert m assert int(m.group(1)) == revcount + 1 + + # This doesn't test astropy_helpers.get_helpers.update_git_devstr directly + # since a copy of that function is made in packagename.version (so that it + # can work without astropy_helpers installed). In order to get test + # coverage on the actual astropy_helpers copy of that function just call it + # directly and compare to the value in packagename + from astropy_helpers.git_helpers import update_git_devstr + + newversion = update_git_devstr(version, path=str(testpackage)) + assert newversion == packagename.__version__ From 89d400fcdcb93c2e6938c1ff1b20c196b41ad638 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 18:54:47 -0400 Subject: [PATCH 24/72] setuptools_boostrap was removed; instead we can just import ah_bootstrap for the same effect --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 45595ab7..51da0fbb 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # Licensed under a 3-clause BSD style license - see LICENSE.rst -import setuptools_bootstrap +import ah_bootstrap import pkg_resources from setuptools import setup from astropy_helpers.version_helpers import generate_version_py From 5d91660323c14b7892e926e2f87179c3fc755884 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 14 Mar 2014 19:52:18 -0400 Subject: [PATCH 25/72] For some reason this is *required* on Python 3.3, where previously it was not. I don't know why since the module's loader should presumably already know the module's name. But so we must... --- astropy_helpers/version_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astropy_helpers/version_helpers.py b/astropy_helpers/version_helpers.py index 95051510..42e218c0 100644 --- a/astropy_helpers/version_helpers.py +++ b/astropy_helpers/version_helpers.py @@ -99,7 +99,8 @@ def _get_version_py_str(packagename, version, release, debug, uses_git=True): if uses_git: loader = pkgutil.get_loader(git_helpers) - source_lines = (loader.get_source() or '').splitlines() + source = loader.get_source(git_helpers.__name__) or '' + source_lines = source.splitlines() if not source_lines: log.warn('Cannot get source code for astropy_helpers.git_helpers; ' 'git support disabled.') From af745754c32dfd3993ce6ecc0ee3e1161aeaf497 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Mon, 17 Mar 2014 09:39:06 -0400 Subject: [PATCH 26/72] Add tox.ini to run the tests on Python 2.6, 2.7, 3.2, and 3.3 (3.1 is skipped due to flagging support from the various tools involved--all the more reason for us to start deprecating 3.1... --- setup.cfg | 4 +++- tox.ini | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tox.ini diff --git a/setup.cfg b/setup.cfg index 6d5f24bf..e80c9cc9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,4 @@ [pytest] -norecursedirs = astropy_helpers/tests/testpackage +norecursedirs = + .tox + astropy_helpers/tests/testpackage diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..09f94820 --- /dev/null +++ b/tox.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py26,py27,py32,py33 + +[testenv] +deps = + pytest + numpy + Cython +commands = py.test +sitepackages = False From c91e5eb2d82a6ee79606913821f1f02356128e4c Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Tue, 18 Mar 2014 12:24:39 -0400 Subject: [PATCH 27/72] Ensure test functions begin with 'test_' so that the 'testpackage' fixture isn't treated as a test --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index e80c9cc9..0cbb9b32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,3 +2,4 @@ norecursedirs = .tox astropy_helpers/tests/testpackage +python_functions = test_ From 5bd4ba3f39b8804330b7c4b8c2d20716cef65a66 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Tue, 18 Mar 2014 15:41:43 -0400 Subject: [PATCH 28/72] Add some configuration for test coverage --- .coveragerc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..a7bdfb49 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,21 @@ +[run] +source = + astropy_helpers + ah_bootstrap + +omit = astropy_helpers/tests* + +[report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about packages we have installed + except ImportError + + # Don't complain if tests don't hit assertions + raise AssertionError + raise NotImplementedError + + # Don't complain about script hooks + def main\(.*\): From e798a654239eb42eb2710d63d0f3e71a0cc6cddc Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Tue, 18 Mar 2014 15:42:53 -0400 Subject: [PATCH 29/72] Ensure that the main copy of ah_bootstrap.py is always used, and that we are returned to the cwd at the end of a test that uses the testpackage fixture --- astropy_helpers/tests/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index 25fd2f51..a1968941 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -27,7 +27,7 @@ def run_cmd(cmd, args, path=None): @pytest.fixture -def testpackage(tmpdir): +def testpackage(tmpdir, request): """Create a copy of the testpackage repository (containing the package template) in a tempdir and change directories to that temporary copy. @@ -38,6 +38,16 @@ def testpackage(tmpdir): tmp_package = tmpdir.join('testpackage') shutil.copytree(os.path.join(PACKAGE_DIR, 'testpackage'), str(tmp_package)) + + def finalize(old_cwd=os.getcwd()): + os.chdir(old_cwd) + + # Before changing directores import the local ah_boostrap module so that it + # is tested, and *not* the copy that happens to be included in the test + # package + + import ah_bootstrap + os.chdir(str(tmp_package)) if 'packagename' in sys.modules: @@ -48,4 +58,6 @@ def testpackage(tmpdir): sys.path.insert(0, '') + request.addfinalizer(finalize) + return tmp_package From 9131040b8115bffc3e852c3507d128a2060f463d Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Tue, 18 Mar 2014 15:45:29 -0400 Subject: [PATCH 30/72] Add the first test that actually tests ah_bootstrap.use_astropy_helpers directly. Will be adding more tests soon following the same general pattern. --- astropy_helpers/tests/test_ah_bootstrap.py | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 astropy_helpers/tests/test_ah_bootstrap.py diff --git a/astropy_helpers/tests/test_ah_bootstrap.py b/astropy_helpers/tests/test_ah_bootstrap.py new file mode 100644 index 00000000..f68d29af --- /dev/null +++ b/astropy_helpers/tests/test_ah_bootstrap.py @@ -0,0 +1,74 @@ +import os + +from setuptools.sandbox import run_setup + +from . import run_cmd + + +TEST_SETUP_PY = """\ +#!/usr/bin/env python +from __future__ import print_function + +import os +import sys +for k in list(sys.modules): + if k == 'astropy_helpers' or k.startswith('astropy_helpers.'): + del sys.modules[k] + +import ah_bootstrap +ah_bootstrap.use_astropy_helpers({args}) + +import astropy_helpers +print(os.path.abspath(astropy_helpers.__file__)) +""" + + +def test_bootstrap_from_submodule(tmpdir, capsys): + """ + Tests importing astropy_helpers from a submodule in a git repository. + This tests actually performing a fresh clone of the repository without + the submodule initialized, and that importing astropy_helpers in that + context works transparently after calling + `ah_boostrap.use_astropy_helpers`. + """ + + orig_repo = tmpdir.mkdir('orig') + old_cwd = os.getcwd() + + # Ensure ah_bootstrap is imported from the local directory + import ah_bootstrap + + try: + os.chdir(str(orig_repo)) + run_cmd('git', ['init']) + + # Write a test setup.py that uses ah_bootstrap; it also ensures that + # any previous reference to astropy_helpers is first wiped from + # sys.modules + orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args='')) + run_cmd('git', ['add', 'setup.py']) + + # Add our own clone of the astropy_helpers repo as a submodule named + # astropy_helpers + run_cmd('git', ['submodule', 'add', os.path.abspath(old_cwd), + 'astropy_helpers']) + + run_cmd('git', ['commit', '-m', 'test repository']) + + os.chdir(str(tmpdir)) + + # Creates a clone of our test repo in the directory 'clone' + run_cmd('git', ['clone', 'orig', 'clone']) + + os.chdir('clone') + run_setup('setup.py', []) + + stdout, stderr = capsys.readouterr() + path = stdout.strip() + + # Ensure that the astropy_helpers used by the setup.py is the one that + # was imported from git submodule + assert path == str(tmpdir.join('clone', 'astropy_helpers', + 'astropy_helpers', '__init__.py')) + finally: + os.chdir(old_cwd) From cd8331af43ba53bc700873e19e7574cb21579f02 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 14:36:12 -0400 Subject: [PATCH 31/72] Renamed the testpackage submodule to package_template to make it clearer where this is coming from (likewise rename the testpackage fixture to package_template) --- .gitmodules | 4 ++-- astropy_helpers/tests/__init__.py | 8 ++++---- astropy_helpers/tests/{testpackage => package_template} | 0 astropy_helpers/tests/test_git_helpers.py | 4 ++-- setup.cfg | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename astropy_helpers/tests/{testpackage => package_template} (100%) diff --git a/.gitmodules b/.gitmodules index 93825913..22115aa4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "astropy_helpers/tests/testpackage"] - path = astropy_helpers/tests/testpackage +[submodule "astropy_helpers/tests/package_template"] + path = astropy_helpers/tests/package_template url = git@github.com:embray/package-template.git diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index a1968941..4b013b17 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -27,16 +27,16 @@ def run_cmd(cmd, args, path=None): @pytest.fixture -def testpackage(tmpdir, request): - """Create a copy of the testpackage repository (containing the package +def package_template(tmpdir, request): + """Create a copy of the package_template repository (containing the package template) in a tempdir and change directories to that temporary copy. Also ensures that any previous imports of the test package are unloaded from `sys.modules`. """ - tmp_package = tmpdir.join('testpackage') - shutil.copytree(os.path.join(PACKAGE_DIR, 'testpackage'), + tmp_package = tmpdir.join('package_template') + shutil.copytree(os.path.join(PACKAGE_DIR, 'package_template'), str(tmp_package)) def finalize(old_cwd=os.getcwd()): diff --git a/astropy_helpers/tests/testpackage b/astropy_helpers/tests/package_template similarity index 100% rename from astropy_helpers/tests/testpackage rename to astropy_helpers/tests/package_template diff --git a/astropy_helpers/tests/test_git_helpers.py b/astropy_helpers/tests/test_git_helpers.py index 282d2157..e2a665b7 100644 --- a/astropy_helpers/tests/test_git_helpers.py +++ b/astropy_helpers/tests/test_git_helpers.py @@ -10,7 +10,7 @@ _DEV_VERSION_RE = re.compile(r'\d+\.\d+(?:\.\d+)?\.dev(\d+)') -def test_update_git_devstr(testpackage, capsys): +def test_update_git_devstr(package_template, capsys): """Tests that the commit number in the package's version string updates after git commits even without re-running setup.py. """ @@ -50,5 +50,5 @@ def test_update_git_devstr(testpackage, capsys): # directly and compare to the value in packagename from astropy_helpers.git_helpers import update_git_devstr - newversion = update_git_devstr(version, path=str(testpackage)) + newversion = update_git_devstr(version, path=str(package_template)) assert newversion == packagename.__version__ diff --git a/setup.cfg b/setup.cfg index 0cbb9b32..780a8283 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [pytest] norecursedirs = .tox - astropy_helpers/tests/testpackage + astropy_helpers/tests/package_template python_functions = test_ From 004c7e8c7787a463e43e08093b5bc89a5d89f67d Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 14:37:23 -0400 Subject: [PATCH 32/72] Add global variables to ah_bootstrap for DIST_NAME and PACKAGE_NAME--this is primarily useful for testing ah_boostrap itself with other packages, but it could also be used to make it easy to tweak ah_boostrap for bootstrapping other packages in a similiar manner... --- ah_bootstrap.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 6eb18693..bf342198 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -45,10 +45,11 @@ from setuptools import Distribution # TODO: Maybe enable checking for a specific version of astropy_helpers? +DIST_NAME = 'astropy-helpers' +PACKAGE_NAME = 'astropy_helpers' -def use_astropy_helpers(path='astropy_helpers', download_if_needed=True, - index_url=None): +def use_astropy_helpers(path=None, download_if_needed=True, index_url=None): """ Ensure that the `astropy_helpers` module is available and is importable. This supports automatic submodule initialization if astropy_helpers is @@ -84,6 +85,9 @@ def use_astropy_helpers(path='astropy_helpers', download_if_needed=True, main PyPI server. """ + if path is None: + path = PACKAGE_NAME + if not isinstance(path, _str_types): if path is not None: raise TypeError('path must be a string or None') @@ -164,8 +168,7 @@ def get_option_dict(self, command_name): opts['allow_hosts'] = ('setup script', allow_hosts) return opts - attrs = {'setup_requires': ['astropy-helpers']} - + attrs = {'setup_requires': [DIST_NAME]} if DEBUG: dist = _Distribution(attrs=attrs) else: From f4b1200fa23baf0221dcacae869101ea75b9cc40 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 14:37:41 -0400 Subject: [PATCH 33/72] Some slightly improved debug output in ah_bootstrap --- ah_bootstrap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index bf342198..1cb0bdeb 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -175,6 +175,9 @@ def get_option_dict(self, command_name): with _silence(): dist = _Distribution(attrs=attrs) except Exception as e: + if DEBUG: + raise + msg = 'Error retrieving astropy helpers from {0}:\n{1}' if find_links: source = find_links[0] @@ -183,7 +186,7 @@ def get_option_dict(self, command_name): else: source = 'PyPI' - raise Exception(msg.format(source, str(e))) + raise Exception(msg.format(source, repr(e))) def _directory_import(path, download_if_needed, is_submodule=None): From 0abf78390aba9d5a50f8809cf15ac072cfd1985a Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 14:40:59 -0400 Subject: [PATCH 34/72] Added a new fixture called testpackage (different from the previous fixture of the same name) that creates a very simple package called _astropy_helpers_test_. It's meant as a stand-in for astropy_helpers itself when testing ah_bootstrap. Testing ah_bootstrap itself was difficult since by its very nature its purpose is to make things so that astropy_helpers can be imported. But astropy_helpers is the package under test and is already imported. We could just use a separate Python interpreter but that makes collecting test coverage data difficult. So the solution is to use a simpler stand-in for astropy_helpers which is good enough for testing the functionality of ah_bootstrap. --- astropy_helpers/tests/__init__.py | 43 ++++++++++ astropy_helpers/tests/test_ah_bootstrap.py | 98 ++++++++++++++++++---- 2 files changed, 124 insertions(+), 17 deletions(-) diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index 4b013b17..96f6bf5a 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -3,6 +3,8 @@ import subprocess as sp import sys +from setuptools.sandbox import run_setup + import pytest PACKAGE_DIR = os.path.dirname(__file__) @@ -61,3 +63,44 @@ def finalize(old_cwd=os.getcwd()): request.addfinalizer(finalize) return tmp_package + + +TEST_PACKAGE_SETUP_PY = """\ +#!/usr/bin/env python + +from setuptools import setup + +setup(name='astropy-helpers-test', version='0.0', + packages=['_astropy_helpers_test_'], + zip_safe=False) +""" + + +@pytest.fixture +def testpackage(tmpdir): + """ + This fixture creates a simplified package called _astropy_helpers_test_ + used primarily for testing ah_boostrap, but without using the + astropy_helpers package directly and getting it confused with the + astropy_helpers package already under test. + """ + + old_cwd = os.path.abspath(os.getcwd()) + source = tmpdir.mkdir('testpkg') + + os.chdir(str(source)) + try: + + source.mkdir('_astropy_helpers_test_') + source.ensure('_astropy_helpers_test_', '__init__.py') + source.join('setup.py').write(TEST_PACKAGE_SETUP_PY) + + # Make the new test package into a git repo + run_cmd('git', ['init']) + run_cmd('git', ['add', '--all']) + run_cmd('git', ['commit', '-m', 'test package']) + + finally: + os.chdir(old_cwd) + + return source diff --git a/astropy_helpers/tests/test_ah_bootstrap.py b/astropy_helpers/tests/test_ah_bootstrap.py index f68d29af..f02c5d92 100644 --- a/astropy_helpers/tests/test_ah_bootstrap.py +++ b/astropy_helpers/tests/test_ah_bootstrap.py @@ -1,8 +1,11 @@ +import glob import os +import textwrap from setuptools.sandbox import run_setup -from . import run_cmd +from . import run_cmd, testpackage +from ..utils import silence TEST_SETUP_PY = """\ @@ -11,24 +14,30 @@ import os import sys -for k in list(sys.modules): - if k == 'astropy_helpers' or k.startswith('astropy_helpers.'): - del sys.modules[k] import ah_bootstrap -ah_bootstrap.use_astropy_helpers({args}) - -import astropy_helpers -print(os.path.abspath(astropy_helpers.__file__)) +# reset the name of the package installed by ah_boostrap to +# _astropy_helpers_test_--this will prevent any confusion by pkg_resources with +# any already installed packages named astropy_helpers +ah_bootstrap.DIST_NAME = 'astropy-helpers-test' +ah_bootstrap.PACKAGE_NAME = '_astropy_helpers_test_' +try: + ah_bootstrap.use_astropy_helpers({args}) +finally: + ah_bootstrap.DIST_NAME = 'astropy-helpers' + ah_bootstrap.PACKAGE_NAME = 'astropy_helpers' + +import _astropy_helpers_test_ +print(os.path.abspath(_astropy_helpers_test_.__file__)) """ -def test_bootstrap_from_submodule(tmpdir, capsys): +def test_bootstrap_from_submodule(tmpdir, testpackage, capsys): """ - Tests importing astropy_helpers from a submodule in a git repository. - This tests actually performing a fresh clone of the repository without - the submodule initialized, and that importing astropy_helpers in that - context works transparently after calling + Tests importing _astropy_helpers_test_ from a submodule in a git + repository. This tests actually performing a fresh clone of the repository + without the submodule initialized, and that importing astropy_helpers in + that context works transparently after calling `ah_boostrap.use_astropy_helpers`. """ @@ -50,8 +59,8 @@ def test_bootstrap_from_submodule(tmpdir, capsys): # Add our own clone of the astropy_helpers repo as a submodule named # astropy_helpers - run_cmd('git', ['submodule', 'add', os.path.abspath(old_cwd), - 'astropy_helpers']) + run_cmd('git', ['submodule', 'add', str(testpackage), + '_astropy_helpers_test_']) run_cmd('git', ['commit', '-m', 'test repository']) @@ -61,6 +70,7 @@ def test_bootstrap_from_submodule(tmpdir, capsys): run_cmd('git', ['clone', 'orig', 'clone']) os.chdir('clone') + run_setup('setup.py', []) stdout, stderr = capsys.readouterr() @@ -68,7 +78,61 @@ def test_bootstrap_from_submodule(tmpdir, capsys): # Ensure that the astropy_helpers used by the setup.py is the one that # was imported from git submodule - assert path == str(tmpdir.join('clone', 'astropy_helpers', - 'astropy_helpers', '__init__.py')) + assert path == str(tmpdir.join('clone', '_astropy_helpers_test_', + '_astropy_helpers_test_', + '__init__.py')) + finally: + os.chdir(old_cwd) + + +def test_download_if_needed(tmpdir, testpackage, capsys): + """ + Tests the case where astropy_helpers was not actually included in a + package, or is otherwise missing, and we need to "download" it. + + This does not test actually downloading from the internet--this is normally + done through setuptools' easy_install command which can also install from a + source archive. From the point of view of ah_boostrap the two actions are + equivalent, so we can just as easily simulate this by providing a setup.cfg + giving the path to a source archive to "download" (as though it were a + URL). + """ + + source = tmpdir.mkdir('source') + old_cwd = os.getcwd() + + # Ensure ah_bootstrap is imported from the local directory + import ah_bootstrap + + os.chdir(str(testpackage)) + # Make a source distribution of the test package + with silence(): + run_setup('setup.py', ['sdist', '--dist-dir=dist', + '--formats=gztar']) + + dist_dir = testpackage.join('dist') + + os.chdir(str(source)) + try: + source.join('setup.py').write(TEST_SETUP_PY.format(args='')) + source.join('setup.cfg').write(textwrap.dedent("""\ + [easy_install] + find_links = {find_links} + """.format(find_links=str(dist_dir)))) + + run_setup('setup.py', []) + + stdout, stderr = capsys.readouterr() + path = stdout.strip() + + # easy_install should have worked by 'installing' astropy_helpers as a + # .egg in the current directory + eggs = glob.glob('*.egg') + assert eggs + egg = source.join(eggs[0]) + assert os.path.isdir(str(egg)) + + assert path == str(egg.join('_astropy_helpers_test_', + '__init__.pyc')) finally: os.chdir(old_cwd) From 2aa665bd1b061a4dedc9ee0469b1c30ec09bbda0 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 14:41:32 -0400 Subject: [PATCH 35/72] A couple workarounds for issues with distutils--ensure that the distutils log is always reset to its default threshold. And a workaround for a bug in setuptools... --- astropy_helpers/tests/__init__.py | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index 96f6bf5a..d12d0cdf 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -28,6 +28,18 @@ def run_cmd(cmd, args, path=None): return streams + (p.returncode,) +@pytest.fixture(scope='function', autouse=True) +def reset_distutils_log(): + """ + This is a setup/teardown fixture that ensures the log-level of the + distutils log is always set to a default of WARN, since different + settings could affect tests that check the contents of stdout. + """ + + from distutils import log + log.set_threshold(log.WARN) + + @pytest.fixture def package_template(tmpdir, request): """Create a copy of the package_template repository (containing the package @@ -104,3 +116,37 @@ def testpackage(tmpdir): os.chdir(old_cwd) return source + + +# Ugly workaround: +# setuptools includes a copy of tempfile.TemporaryDirectory for older Python +# versions that do not have that class. But the copy included in setuptools is +# affected by this bug: http://bugs.python.org/issue10188 +# Patch setuptools' TemporaryDirectory so that it doesn't crash on shutdown +class TemporaryDirectory(object): + """" + Very simple temporary directory context manager. + Will try to delete afterward, but will also ignore OS and similar + errors on deletion. + """ + def __init__(self): + import tempfile + self.name = None # Handle mkdtemp raising an exception + self.name = tempfile.mkdtemp() + + def __enter__(self): + return self.name + + def __exit__(self, exctype, excvalue, exctrace): + import shutil + try: + shutil.rmtree(self.name, True) + except OSError: #removal errors are not the only possible + pass + self.name = None + +try: + from tempfile import TemporaryDirectory +except ImportError: + import setuptools.py31compat + setuptools.py31compat.TemporaryDirectory = TemporaryDirectory From 750456de1de9fc3129d0de3f85c608092bed79d8 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 15:56:31 -0400 Subject: [PATCH 36/72] Add another test for ah_bootstrap, this time for bootstrapping from a source archive. --- astropy_helpers/tests/test_ah_bootstrap.py | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/astropy_helpers/tests/test_ah_bootstrap.py b/astropy_helpers/tests/test_ah_bootstrap.py index f02c5d92..fa38967b 100644 --- a/astropy_helpers/tests/test_ah_bootstrap.py +++ b/astropy_helpers/tests/test_ah_bootstrap.py @@ -1,5 +1,6 @@ import glob import os +import shutil import textwrap from setuptools.sandbox import run_setup @@ -85,6 +86,57 @@ def test_bootstrap_from_submodule(tmpdir, testpackage, capsys): os.chdir(old_cwd) +def test_bootstrap_from_archive(tmpdir, testpackage, capsys): + """ + Tests importing _astropy_helpers_test_ from a .tar.gz source archive + shipped alongside the package that uses it. + """ + + orig_repo = tmpdir.mkdir('orig') + old_cwd = os.getcwd() + + # Ensure ah_bootstrap is imported from the local directory + import ah_bootstrap + + os.chdir(str(testpackage)) + # Make a source distribution of the test package + with silence(): + run_setup('setup.py', ['sdist', '--dist-dir=dist', + '--formats=gztar']) + + dist_dir = testpackage.join('dist') + archives = glob.glob(str(dist_dir.join('*.tar.gz'))) + assert len(archives) == 1 + dist_file = archives[0] + + try: + os.chdir(str(orig_repo)) + shutil.copy(dist_file, str(orig_repo)) + + # Write a test setup.py that uses ah_bootstrap; it also ensures that + # any previous reference to astropy_helpers is first wiped from + # sys.modules + args = 'path={0!r}'.format(os.path.basename(dist_file)) + orig_repo.join('setup.py').write(TEST_SETUP_PY.format(args=args)) + + run_setup('setup.py', []) + + stdout, stderr = capsys.readouterr() + path = stdout.strip() + + # Installation from the .tar.gz should have resulted in a .egg + # directory that the _astropy_helpers_test_ package was imported from + eggs = glob.glob('*.egg') + assert eggs + egg = orig_repo.join(eggs[0]) + assert os.path.isdir(str(egg)) + + assert path == str(egg.join('_astropy_helpers_test_', + '__init__.pyc')) + finally: + os.chdir(old_cwd) + + def test_download_if_needed(tmpdir, testpackage, capsys): """ Tests the case where astropy_helpers was not actually included in a From 784bbf96377fd55eb2936cc1cbe9cafba4c782ac Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Wed, 19 Mar 2014 16:14:50 -0400 Subject: [PATCH 37/72] A few fixes to the tests on Python 2.6 and 3.x --- ah_bootstrap.py | 10 ++++++---- astropy_helpers/tests/test_ah_bootstrap.py | 16 +++++++++++----- astropy_helpers/utils.py | 2 ++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 1cb0bdeb..e912dc50 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -263,16 +263,16 @@ def _check_submodule(path): def _update_submodule(submodule, status): - if status == ' ': + if status == b' ': # The submodule is up to date; no action necessary return - elif status == '-': + elif status == b'-': cmd = ['update', '--init'] log.info('Initializing submodule {0!r}'.format(submodule)) - elif status == '+': + elif status == b'+': cmd = ['update'] log.info('Updating submodule {0!r}'.format(submodule)) - elif status == 'U': + elif status == b'U': raise _AHBoostrapSystemExit( 'Error: Submodule {0} contains unresolved merge conflicts. ' 'Please complete or abandon any changes in the submodule so that ' @@ -305,6 +305,8 @@ def _update_submodule(submodule, status): class _DummyFile(object): """A noop writeable object.""" + errors = '' # Required for Python 3.x + def write(self, s): pass diff --git a/astropy_helpers/tests/test_ah_bootstrap.py b/astropy_helpers/tests/test_ah_bootstrap.py index fa38967b..f5ee421b 100644 --- a/astropy_helpers/tests/test_ah_bootstrap.py +++ b/astropy_helpers/tests/test_ah_bootstrap.py @@ -29,7 +29,9 @@ ah_bootstrap.PACKAGE_NAME = 'astropy_helpers' import _astropy_helpers_test_ -print(os.path.abspath(_astropy_helpers_test_.__file__)) +filename = os.path.abspath(_astropy_helpers_test_.__file__) +filename = filename.replace('.pyc', '.py') # More consistent this way +print(filename) """ @@ -122,7 +124,7 @@ def test_bootstrap_from_archive(tmpdir, testpackage, capsys): run_setup('setup.py', []) stdout, stderr = capsys.readouterr() - path = stdout.strip() + path = stdout.splitlines()[-1].strip() # Installation from the .tar.gz should have resulted in a .egg # directory that the _astropy_helpers_test_ package was imported from @@ -132,7 +134,7 @@ def test_bootstrap_from_archive(tmpdir, testpackage, capsys): assert os.path.isdir(str(egg)) assert path == str(egg.join('_astropy_helpers_test_', - '__init__.pyc')) + '__init__.py')) finally: os.chdir(old_cwd) @@ -175,7 +177,11 @@ def test_download_if_needed(tmpdir, testpackage, capsys): run_setup('setup.py', []) stdout, stderr = capsys.readouterr() - path = stdout.strip() + + # Just take the last line--on Python 2.6 distutils logs warning + # messages to stdout instead of stderr, causing them to be mixed up + # with our expected output + path = stdout.splitlines()[-1].strip() # easy_install should have worked by 'installing' astropy_helpers as a # .egg in the current directory @@ -185,6 +191,6 @@ def test_download_if_needed(tmpdir, testpackage, capsys): assert os.path.isdir(str(egg)) assert path == str(egg.join('_astropy_helpers_test_', - '__init__.pyc')) + '__init__.py')) finally: os.chdir(old_cwd) diff --git a/astropy_helpers/utils.py b/astropy_helpers/utils.py index 04708086..bb05fb8c 100644 --- a/astropy_helpers/utils.py +++ b/astropy_helpers/utils.py @@ -19,6 +19,8 @@ class _DummyFile(object): """A noop writeable object.""" + errors = '' # Required for Python 3.x + def write(self, s): pass From a9eea892cd10d229e9675670f3c022a362899f83 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Thu, 20 Mar 2014 16:16:37 -0400 Subject: [PATCH 38/72] Turns out the cause of this problem wasn't exactly what I thought it was. We don't have to reimplement TemporaryDirectory. I'm still not entirely clear on *what* causes this but it's not what I thought. --- astropy_helpers/tests/__init__.py | 37 +++++-------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/astropy_helpers/tests/__init__.py b/astropy_helpers/tests/__init__.py index d12d0cdf..105c2ab9 100644 --- a/astropy_helpers/tests/__init__.py +++ b/astropy_helpers/tests/__init__.py @@ -118,35 +118,8 @@ def testpackage(tmpdir): return source -# Ugly workaround: -# setuptools includes a copy of tempfile.TemporaryDirectory for older Python -# versions that do not have that class. But the copy included in setuptools is -# affected by this bug: http://bugs.python.org/issue10188 -# Patch setuptools' TemporaryDirectory so that it doesn't crash on shutdown -class TemporaryDirectory(object): - """" - Very simple temporary directory context manager. - Will try to delete afterward, but will also ignore OS and similar - errors on deletion. - """ - def __init__(self): - import tempfile - self.name = None # Handle mkdtemp raising an exception - self.name = tempfile.mkdtemp() - - def __enter__(self): - return self.name - - def __exit__(self, exctype, excvalue, exctrace): - import shutil - try: - shutil.rmtree(self.name, True) - except OSError: #removal errors are not the only possible - pass - self.name = None - -try: - from tempfile import TemporaryDirectory -except ImportError: - import setuptools.py31compat - setuptools.py31compat.TemporaryDirectory = TemporaryDirectory +# Ugly workaround +# Note sure exactly why, but there is some weird interaction between setuptools +# entry points and the way run_setup messes with sys.modules that causes this +# module go out out of scope during the tests; importing it here prevents that +import setuptools.py31compat From 288ac6ab1491321d9eca6f8aea04b47b935fe7ce Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Fri, 21 Mar 2014 15:13:12 -0400 Subject: [PATCH 39/72] Some changes to how packages are searched; the srcdir for get_package_info does not return the package info for a single specific package. Rather it it returns info for all packages under the given path (by default '.', but this can work for source trees that don't place the Python packages immediately at the root of the source (though that would generally be advisable). It also allows excluding packages and supports that even for extension modules. --- astropy_helpers/setup_helpers.py | 29 ++++++++++++++++++----------- setup.py | 5 +++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index 0f26e7b0..0b2facfd 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -946,7 +946,7 @@ def update_package_files(srcdir, extensions, package_data, packagenames, package_dirs.update(info['package_dir']) -def get_package_info(srcdir): +def get_package_info(srcdir='.', exclude=()): """ Collates all of the information for building all subpackages subpackages and returns a dictionary of keyword arguments that can @@ -993,13 +993,13 @@ def get_package_info(srcdir): skip_2to3 = [] # Use the find_packages tool to locate all packages and modules - packages = filter_packages(find_packages()) + packages = filter_packages(find_packages(srcdir, exclude=exclude)) # For each of the setup_package.py modules, extract any # information that is needed to install them. The build options # are extracted first, so that their values will be available in # subsequent calls to `get_extensions`, etc. - for setuppkg in iter_setup_packages(srcdir): + for setuppkg in iter_setup_packages(srcdir, packages): if hasattr(setuppkg, 'get_build_options'): options = setuppkg.get_build_options() for option in options: @@ -1016,7 +1016,7 @@ def get_package_info(srcdir): skip_2to3.append( os.path.dirname(setuppkg.__file__)) - for setuppkg in iter_setup_packages(srcdir): + for setuppkg in iter_setup_packages(srcdir, packages): # get_extensions must include any Cython extensions by their .pyx # filename. if hasattr(setuppkg, 'get_extensions'): @@ -1051,7 +1051,7 @@ def get_package_info(srcdir): } -def iter_setup_packages(srcdir): +def iter_setup_packages(srcdir, packages): """ A generator that finds and imports all of the ``setup_package.py`` modules in the source packages. @@ -1062,9 +1062,13 @@ def iter_setup_packages(srcdir): `modname` is the module name for the ``setup_package.py`` modules. """ - for root, dirs, files in walk_skip_hidden(srcdir): - if 'setup_package.py' in files: - filename = os.path.join(root, 'setup_package.py') + + for packagename in packages: + package_parts = packagename.split('.') + package_path = os.path.join(srcdir, *package_parts) + setup_package = os.path.join(package_path, 'setup_package.py') + + if os.path.isfile(setup_package): module = import_file(filename) yield module @@ -1083,12 +1087,15 @@ def iter_pyx_files(srcdir): """ for dirpath, dirnames, filenames in walk_skip_hidden(srcdir): - modbase = dirpath.replace(os.sep, '.') + srcdir_parts = srcdir.split(os.sep) + packagename_parts = dirpath.split(os.sep)[len(srcdir_parts):] + packagename = '.'.join(packagename_parts) + for fn in filenames: if fn.endswith('.pyx'): - fullfn = os.path.join(dirpath, fn) + fullfn = os.path.relpath(os.path.join(dirpath, fn)) # Package must match file name - extmod = modbase + '.' + fn[:-4] + extmod = '.'.join([packagename, fn[:-4]]) yield (extmod, fullfn) diff --git a/setup.py b/setup.py index 51da0fbb..228c4f93 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import ah_bootstrap import pkg_resources from setuptools import setup +from astropy_helpers.setup_helpers import register_commands, get_package_info from astropy_helpers.version_helpers import generate_version_py NAME = 'astropy_helpers' @@ -27,7 +28,7 @@ long_description='', download_url='{0}/astropy-{1}.tar.gz'.format(DOWNLOAD_BASE_URL, VERSION), classifiers=[], - cmdclass={}, + cmdclass=register_commands(NAME, VERSION, RELEASE), zip_safe=False, - packages=[NAME] + **get_package_info(exclude=['astropy_helpers.tests']) ) From 1e8ca3ed8acc053b575757efb84534714844c9c9 Mon Sep 17 00:00:00 2001 From: "Erik M. Bray" Date: Tue, 25 Mar 2014 18:27:41 -0400 Subject: [PATCH 40/72] Initial import of astropy.sphinx into astropy_helpers.sphinx--mostly the same except for a few small tweaks to get it working under the name 'astropy_helpers' --- astropy_helpers/compat/__init__.py | 0 .../compat/_subprocess_py2/__init__.py | 38 + astropy_helpers/compat/subprocess.py | 19 + astropy_helpers/setup_helpers.py | 2 +- astropy_helpers/sphinx/__init__.py | 6 + astropy_helpers/sphinx/conf.py | 314 +++++++ astropy_helpers/sphinx/ext/__init__.py | 9 + .../sphinx/ext/astropyautosummary.py | 92 ++ astropy_helpers/sphinx/ext/automodapi.py | 305 +++++++ astropy_helpers/sphinx/ext/automodsumm.py | 550 +++++++++++ astropy_helpers/sphinx/ext/changelog_links.py | 66 ++ astropy_helpers/sphinx/ext/comment_eater.py | 158 ++++ .../sphinx/ext/compiler_unparse.py | 860 ++++++++++++++++++ astropy_helpers/sphinx/ext/docscrape.py | 508 +++++++++++ .../sphinx/ext/docscrape_sphinx.py | 227 +++++ astropy_helpers/sphinx/ext/doctest.py | 33 + astropy_helpers/sphinx/ext/edit_on_github.py | 164 ++++ astropy_helpers/sphinx/ext/numpydoc.py | 169 ++++ astropy_helpers/sphinx/ext/phantom_import.py | 162 ++++ .../ext/templates/autosummary_core/base.rst | 10 + .../ext/templates/autosummary_core/class.rst | 65 ++ .../ext/templates/autosummary_core/module.rst | 41 + astropy_helpers/sphinx/ext/tests/__init__.py | 0 .../sphinx/ext/tests/test_automodapi.py | 300 ++++++ .../sphinx/ext/tests/test_automodsumm.py | 75 ++ .../sphinx/ext/tests/test_utils.py | 29 + astropy_helpers/sphinx/ext/tocdepthfix.py | 18 + astropy_helpers/sphinx/ext/traitsdoc.py | 140 +++ astropy_helpers/sphinx/ext/utils.py | 65 ++ astropy_helpers/sphinx/ext/viewcode.py | 212 +++++ astropy_helpers/sphinx/setup_package.py | 9 + .../themes/bootstrap-astropy/globaltoc.html | 3 + .../themes/bootstrap-astropy/layout.html | 94 ++ .../themes/bootstrap-astropy/localtoc.html | 3 + .../themes/bootstrap-astropy/searchbox.html | 7 + .../static/astropy_linkout_20.png | Bin 0 -> 1725 bytes .../bootstrap-astropy/static/astropy_logo.ico | Bin 0 -> 1150 bytes .../static/astropy_logo_32.png | Bin 0 -> 1884 bytes .../static/bootstrap-astropy.css | 573 ++++++++++++ .../bootstrap-astropy/static/sidebar.js | 160 ++++ .../themes/bootstrap-astropy/theme.conf | 10 + 41 files changed, 5495 insertions(+), 1 deletion(-) create mode 100644 astropy_helpers/compat/__init__.py create mode 100644 astropy_helpers/compat/_subprocess_py2/__init__.py create mode 100644 astropy_helpers/compat/subprocess.py create mode 100644 astropy_helpers/sphinx/__init__.py create mode 100644 astropy_helpers/sphinx/conf.py create mode 100644 astropy_helpers/sphinx/ext/__init__.py create mode 100644 astropy_helpers/sphinx/ext/astropyautosummary.py create mode 100644 astropy_helpers/sphinx/ext/automodapi.py create mode 100644 astropy_helpers/sphinx/ext/automodsumm.py create mode 100644 astropy_helpers/sphinx/ext/changelog_links.py create mode 100644 astropy_helpers/sphinx/ext/comment_eater.py create mode 100644 astropy_helpers/sphinx/ext/compiler_unparse.py create mode 100644 astropy_helpers/sphinx/ext/docscrape.py create mode 100644 astropy_helpers/sphinx/ext/docscrape_sphinx.py create mode 100644 astropy_helpers/sphinx/ext/doctest.py create mode 100644 astropy_helpers/sphinx/ext/edit_on_github.py create mode 100644 astropy_helpers/sphinx/ext/numpydoc.py create mode 100644 astropy_helpers/sphinx/ext/phantom_import.py create mode 100644 astropy_helpers/sphinx/ext/templates/autosummary_core/base.rst create mode 100644 astropy_helpers/sphinx/ext/templates/autosummary_core/class.rst create mode 100644 astropy_helpers/sphinx/ext/templates/autosummary_core/module.rst create mode 100644 astropy_helpers/sphinx/ext/tests/__init__.py create mode 100644 astropy_helpers/sphinx/ext/tests/test_automodapi.py create mode 100644 astropy_helpers/sphinx/ext/tests/test_automodsumm.py create mode 100644 astropy_helpers/sphinx/ext/tests/test_utils.py create mode 100644 astropy_helpers/sphinx/ext/tocdepthfix.py create mode 100644 astropy_helpers/sphinx/ext/traitsdoc.py create mode 100644 astropy_helpers/sphinx/ext/utils.py create mode 100644 astropy_helpers/sphinx/ext/viewcode.py create mode 100644 astropy_helpers/sphinx/setup_package.py create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/globaltoc.html create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/layout.html create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/localtoc.html create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/searchbox.html create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_linkout_20.png create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo.ico create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo_32.png create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/static/bootstrap-astropy.css create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/static/sidebar.js create mode 100644 astropy_helpers/sphinx/themes/bootstrap-astropy/theme.conf diff --git a/astropy_helpers/compat/__init__.py b/astropy_helpers/compat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/astropy_helpers/compat/_subprocess_py2/__init__.py b/astropy_helpers/compat/_subprocess_py2/__init__.py new file mode 100644 index 00000000..a14df41b --- /dev/null +++ b/astropy_helpers/compat/_subprocess_py2/__init__.py @@ -0,0 +1,38 @@ +from __future__ import absolute_import + +from subprocess import * + + +def check_output(*popenargs, **kwargs): + r"""Run command with arguments and return its output as a byte + string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the + returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example:: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT.:: + + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) + return output diff --git a/astropy_helpers/compat/subprocess.py b/astropy_helpers/compat/subprocess.py new file mode 100644 index 00000000..93a388b5 --- /dev/null +++ b/astropy_helpers/compat/subprocess.py @@ -0,0 +1,19 @@ +""" +A replacement wrapper around the subprocess module that adds +check_output (which was only added to Python in 2.7. + +Instead of importing subprocess, other modules should use this as follows:: + + from astropy.utils.compat import subprocess + +This module is safe to import from anywhere within astropy. +""" +from __future__ import absolute_import, print_function + +import subprocess + +# python2.7 and later provide a check_output method +if not hasattr(subprocess, 'check_output'): + from ._subprocess_py2 import check_output + +from subprocess import * diff --git a/astropy_helpers/setup_helpers.py b/astropy_helpers/setup_helpers.py index 0b2facfd..6206c9ec 100644 --- a/astropy_helpers/setup_helpers.py +++ b/astropy_helpers/setup_helpers.py @@ -1069,7 +1069,7 @@ def iter_setup_packages(srcdir, packages): setup_package = os.path.join(package_path, 'setup_package.py') if os.path.isfile(setup_package): - module = import_file(filename) + module = import_file(setup_package) yield module diff --git a/astropy_helpers/sphinx/__init__.py b/astropy_helpers/sphinx/__init__.py new file mode 100644 index 00000000..e446ac01 --- /dev/null +++ b/astropy_helpers/sphinx/__init__.py @@ -0,0 +1,6 @@ +""" +This package contains utilities and extensions for the Astropy sphinx +documentation. In particular, the `astropy.sphinx.conf` should be imported by +the sphinx ``conf.py`` file for affiliated packages that wish to make use of +the Astropy documentation format. +""" diff --git a/astropy_helpers/sphinx/conf.py b/astropy_helpers/sphinx/conf.py new file mode 100644 index 00000000..5bcb91c6 --- /dev/null +++ b/astropy_helpers/sphinx/conf.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# +# Astropy shared Sphinx settings. These settings are shared between +# astropy itself and affiliated packages. +# +# Note that not all possible configuration values are present in this file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import re +import warnings + +from os import path +from distutils.version import LooseVersion + +from ..compat import subprocess + + +# -- General configuration ---------------------------------------------------- + +# Some of the docs require the autodoc special-members option, in 1.1. +# If using graphviz 2.30 or later, Sphinx < 1.2b2 will not work with +# it. Unfortunately, there are other problems with Sphinx 1.2b2, so +# we need to use "dev" until a release is made post 1.2b2. If +# affiliated packages don't want this automatic determination, they +# may simply override needs_sphinx in their local conf.py. + +def get_graphviz_version(): + try: + output = subprocess.check_output( + ['dot', '-V'], stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True) + except subprocess.CalledProcessError: + return '0' + tokens = output.split() + for token in tokens: + if re.match(b'[0-9.]*', token): + return token.decode('ascii') + return '0' + +graphviz_found = LooseVersion(get_graphviz_version()) +graphviz_broken = LooseVersion('0.30') + +if graphviz_found >= graphviz_broken: + needs_sphinx = '1.2' +else: + needs_sphinx = '1.1' + +# Configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('http://docs.python.org/', None), + 'numpy': ('http://docs.scipy.org/doc/numpy/', None), + 'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None), + 'matplotlib': ('http://matplotlib.sourceforge.net/', None), + 'astropy': ('http://docs.astropy.org/en/stable/', None), + 'h5py': ('http://docs.h5py.org/en/latest/', None) + } + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# The reST default role (used for this markup: `text`) to use for all +# documents. Set to the "smart" one. +default_role = 'obj' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# This is added to the end of RST files - a good place to put substitutions to +# be used globally. +rst_epilog = """ +.. _Astropy: http://astropy.org +""" + + +# -- Project information ------------------------------------------------------ + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +#pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Settings for extensions and extension options ---------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.pngmath', + 'sphinx.ext.inheritance_diagram', + 'astropy_helpers.sphinx.ext.numpydoc', + 'astropy_helpers.sphinx.ext.astropyautosummary', + 'astropy_helpers.sphinx.ext.automodsumm', + 'astropy_helpers.sphinx.ext.automodapi', + 'astropy_helpers.sphinx.ext.tocdepthfix', + 'astropy_helpers.sphinx.ext.doctest', + 'astropy_helpers.sphinx.ext.changelog_links', + 'astropy_helpers.sphinx.ext.viewcode' # Use patched version of viewcode + ] + +# Above, we use a patched version of viewcode rather than 'sphinx.ext.viewcode' +# This can be changed to the sphinx version once the following issue is fixed +# in sphinx: +# https://bitbucket.org/birkenfeld/sphinx/issue/623/ +# extension-viewcode-fails-with-function + +try: + import matplotlib.sphinxext.plot_directive + extensions += [matplotlib.sphinxext.plot_directive.__name__] +# AttributeError is checked here in case matplotlib is installed but +# Sphinx isn't. Note that this module is imported by the config file +# generator, even if we're not building the docs. +except (ImportError, AttributeError): + warnings.warn( + "matplotlib's plot_directive could not be imported. " + + "Inline plots will not be included in the output") + +# Don't show summaries of the members in each class along with the +# class' docstring +numpydoc_show_class_members = False + +autosummary_generate = True + +automodapi_toctreedirnm = 'api' + + +# -- Options for HTML output ------------------------------------------------- + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [path.abspath(path.join(path.dirname(__file__), 'themes'))] + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'bootstrap-astropy' + +# Custom sidebar templates, maps document names to template names. +html_sidebars = { + '**': ['localtoc.html'], + 'search': [], + 'genindex': [], + 'py-modindex': [], +} + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = 'astropy_logo.ico' # included in the bootstrap-astropy theme + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%d %b %Y' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# -- Options for LaTeX output ------------------------------------------------ + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +latex_use_parts = True + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +latex_preamble = r""" +% Use a more modern-looking monospace font +\usepackage{inconsolata} + +% The enumitem package provides unlimited nesting of lists and enums. +% Sphinx may use this in the future, in which case this can be removed. +% See https://bitbucket.org/birkenfeld/sphinx/issue/777/latex-output-too-deeply-nested +\usepackage{enumitem} +\setlistdepth{15} + +% In the parameters section, place a newline after the Parameters +% header. (This is stolen directly from Numpy's conf.py, since it +% affects Numpy-style docstrings). +\usepackage{expdlist} +\let\latexdescription=\description +\def\description{\latexdescription{}{} \breaklabel} + +% Support the superscript Unicode numbers used by the "unicode" units +% formatter +\DeclareUnicodeCharacter{2070}{\ensuremath{^0}} +\DeclareUnicodeCharacter{00B9}{\ensuremath{^1}} +\DeclareUnicodeCharacter{00B2}{\ensuremath{^2}} +\DeclareUnicodeCharacter{00B3}{\ensuremath{^3}} +\DeclareUnicodeCharacter{2074}{\ensuremath{^4}} +\DeclareUnicodeCharacter{2075}{\ensuremath{^5}} +\DeclareUnicodeCharacter{2076}{\ensuremath{^6}} +\DeclareUnicodeCharacter{2077}{\ensuremath{^7}} +\DeclareUnicodeCharacter{2078}{\ensuremath{^8}} +\DeclareUnicodeCharacter{2079}{\ensuremath{^9}} +\DeclareUnicodeCharacter{207B}{\ensuremath{^-}} +\DeclareUnicodeCharacter{00B0}{\ensuremath{^{\circ}}} +\DeclareUnicodeCharacter{2032}{\ensuremath{^{\prime}}} +\DeclareUnicodeCharacter{2033}{\ensuremath{^{\prime\prime}}} + +% Make the "warning" and "notes" sections use a sans-serif font to +% make them stand out more. +\renewenvironment{notice}[2]{ + \def\py@noticetype{#1} + \csname py@noticestart@#1\endcsname + \textsf{\textbf{#2}} +}{\csname py@noticeend@\py@noticetype\endcsname} +""" + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# -- Options for the linkcheck builder ---------------------------------------- + +# A timeout value, in seconds, for the linkcheck builder +linkcheck_timeout = 60 diff --git a/astropy_helpers/sphinx/ext/__init__.py b/astropy_helpers/sphinx/ext/__init__.py new file mode 100644 index 00000000..bfc22933 --- /dev/null +++ b/astropy_helpers/sphinx/ext/__init__.py @@ -0,0 +1,9 @@ +""" +This package contains the sphinx extensions used by Astropy. + +The `automodsumm` and `automodapi` modules contain extensions used by Astropy +to generate API documentation - see the docstrings for details. The `numpydoc` +module is dervied from the documentation tools numpy and scipy use to parse +docstrings in the numpy/scipy format, and are also used by Astropy. The +other modules are dependencies for `numpydoc`. +""" diff --git a/astropy_helpers/sphinx/ext/astropyautosummary.py b/astropy_helpers/sphinx/ext/astropyautosummary.py new file mode 100644 index 00000000..e42b6323 --- /dev/null +++ b/astropy_helpers/sphinx/ext/astropyautosummary.py @@ -0,0 +1,92 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This sphinx extension builds off of `sphinx.ext.autosummary` to +clean up some issues it presents in the Astropy docs. + +The main issue this fixes is the summary tables getting cut off before the +end of the sentence in some cases. + +""" +import re + +from sphinx.ext.autosummary import Autosummary + +# used in AstropyAutosummary.get_items +_itemsummrex = re.compile(r'^([A-Z].*?\.(?:\s|$))') + + +class AstropyAutosummary(Autosummary): + def get_items(self, names): + """Try to import the given names, and return a list of + ``[(name, signature, summary_string, real_name), ...]``. + """ + from sphinx.ext.autosummary import (get_import_prefixes_from_env, + import_by_name, get_documenter, mangle_signature) + + env = self.state.document.settings.env + + prefixes = get_import_prefixes_from_env(env) + + items = [] + + max_item_chars = 50 + + for name in names: + display_name = name + if name.startswith('~'): + name = name[1:] + display_name = name.split('.')[-1] + + try: + real_name, obj, parent = import_by_name(name, prefixes=prefixes) + except ImportError: + self.warn('[astropyautosummary] failed to import %s' % name) + items.append((name, '', '', name)) + continue + + # NB. using real_name here is important, since Documenters + # handle module prefixes slightly differently + documenter = get_documenter(obj, parent)(self, real_name) + if not documenter.parse_name(): + self.warn('[astropyautosummary] failed to parse name %s' % real_name) + items.append((display_name, '', '', real_name)) + continue + if not documenter.import_object(): + self.warn('[astropyautosummary] failed to import object %s' % real_name) + items.append((display_name, '', '', real_name)) + continue + + # -- Grab the signature + + sig = documenter.format_signature() + if not sig: + sig = '' + else: + max_chars = max(10, max_item_chars - len(display_name)) + sig = mangle_signature(sig, max_chars=max_chars) + sig = sig.replace('*', r'\*') + + # -- Grab the summary + + doc = list(documenter.process_doc(documenter.get_doc())) + + while doc and not doc[0].strip(): + doc.pop(0) + m = _itemsummrex.search(" ".join(doc).strip()) + if m: + summary = m.group(1).strip() + elif doc: + summary = doc[0].strip() + else: + summary = '' + + items.append((display_name, sig, summary, real_name)) + + return items + + +def setup(app): + # need autosummary, of course + app.setup_extension('sphinx.ext.autosummary') + # this replaces the default autosummary with the astropy one + app.add_directive('autosummary', AstropyAutosummary) diff --git a/astropy_helpers/sphinx/ext/automodapi.py b/astropy_helpers/sphinx/ext/automodapi.py new file mode 100644 index 00000000..73302c69 --- /dev/null +++ b/astropy_helpers/sphinx/ext/automodapi.py @@ -0,0 +1,305 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This sphinx extension adds a tools to simplify generating the API +documentation for Astropy packages and affiliated packages. + +====================== +`automodapi` directive +====================== +This directive takes a single argument that must be a module or package. +It will produce a block of documentation that includes the docstring for +the package, an `automodsumm` directive, and an `automod-diagram` if +there are any classes in the module. + +It accepts the following options: + + * ``:no-inheritance-diagram:`` + If present, the inheritance diagram will not be shown even if + the module/package has classes. + + * ``:skip: str`` + This option results in the + specified object being skipped, that is the object will *not* be + included in the generated documentation. This option may appear + any number of times to skip multiple objects. + + * ``:no-main-docstr:`` + If present, the docstring for the module/package will not be generated. + The function and class tables will still be used, however. + + * ``:headings: str`` + Specifies the characters (in one string) used as the heading + levels used for the generated section. This must have at least 2 + characters (any after 2 will be ignored). This also *must* match + the rest of the documentation on this page for sphinx to be + happy. Defaults to "-^", which matches the convention used for + Python's documentation, assuming the automodapi call is inside a + top-level section (which usually uses '='). + + * ``:no-heading:``` + If specified do not create a top level heading for the section. + +This extension also adds two sphinx configuration option: + +* `automodapi_toctreedirnm` + This must be a string that specifies the name of the directory the + automodsumm generated documentation ends up in. This directory path should + be relative to the documentation root (e.g., same place as ``index.rst``). + Defaults to ``'api'``. + +* `automodapi_writereprocessed` + Should be a bool, and if True, will cause `automodapi` to write files with + any ``automodapi`` sections replaced with the content Sphinx processes + after ``automodapi`` has run. The output files are not actually used by + sphinx, so this option is only for figuring out the cause of sphinx warnings + or other debugging. Defaults to `False`. + +""" + +# Implementation note: +# The 'automodapi' directive is not actually implemented as a docutils +# directive. Instead, this extension searches for the 'automodapi' text in +# all sphinx documents, and replaces it where necessary from a template built +# into this extension. This is necessary because automodsumm (and autosummary) +# use the "builder-inited" event, which comes before the directives are +# actually built. + +import inspect +import os +import re +import sys + +from .utils import find_mod_objs + + +automod_templ_modheader = """ +{modname} {pkgormod} +{modhds}{pkgormodhds} + +{automoduleline} +""" + +automod_templ_classes = """ +Classes +{clshds} + +.. automodsumm:: {modname} + :classes-only: + {toctree} + {skips} +""" + +automod_templ_funcs = """ +Functions +{funchds} + +.. automodsumm:: {modname} + :functions-only: + {toctree} + {skips} +""" + +automod_templ_inh = """ +Class Inheritance Diagram +{clsinhsechds} + +.. automod-diagram:: {modname} + :private-bases: +""" + +_automodapirex = re.compile(r'^(?:\s*\.\.\s+automodapi::\s*)([A-Za-z0-9_.]+)' + r'\s*$((?:\n\s+:[a-zA-Z_\-]+:.*$)*)', + flags=re.MULTILINE) +#the last group of the above regex is intended to go into finall with the below +_automodapiargsrex = re.compile(r':([a-zA-Z_\-]+):(.*)$', flags=re.MULTILINE) + + +def automodapi_replace(sourcestr, app, dotoctree=True, docname=None, + warnings=True): + """ + Replaces `sourcestr`'s entries of ".. automdapi::" with the + automodapi template form based on provided options. + + This is used with the sphinx event 'source-read' to replace + `automodapi` entries before sphinx actually processes them, as + automodsumm needs the code to be present to generate stub + documentation. + + Parameters + ---------- + sourcestr : str + The string with sphinx source to be checked for automodapi + replacement. + app : `sphinx.application.Application` + The sphinx application. + dotoctree : bool + If True, a ":toctree:" option will be added in the ".. + automodsumm::" sections of the template, pointing to the + appropriate "generated" directory based on the Astropy convention + (e.g. in ``docs/api``) + docname : str + The name of the file for this `sourcestr` (if known - if not, it + can be None). If not provided and `dotoctree` is True, the + generated files may end up in the wrong place. + warnings : bool + If False, all warnings that would normally be issued are + silenced. + + Returns + ------- + newstr :str + The string with automodapi entries replaced with the correct + sphinx markup. + """ + + spl = _automodapirex.split(sourcestr) + if len(spl) > 1: # automodsumm is in this document + + if dotoctree: + toctreestr = ':toctree: ' + dirnm = app.config.automodapi_toctreedirnm + if not dirnm.endswith(os.sep): + dirnm += os.sep + if docname is not None: + toctreestr += '../' * docname.count('/') + dirnm + else: + toctreestr += dirnm + else: + toctreestr = '' + + newstrs = [spl[0]] + for grp in range(len(spl) // 3): + modnm = spl[grp * 3 + 1] + + #find where this is in the document for warnings + if docname is None: + location = None + else: + location = (docname, spl[0].count('\n')) + + #initialize default options + toskip = [] + inhdiag = maindocstr = top_head = True + hds = '-^' + + #look for actual options + unknownops = [] + for opname, args in _automodapiargsrex.findall(spl[grp * 3 + 2]): + if opname == 'skip': + toskip.append(args.strip()) + elif opname == 'no-inheritance-diagram': + inhdiag = False + elif opname == 'no-main-docstr': + maindocstr = False + elif opname == 'headings': + hds = args + elif opname == 'no-heading': + top_head = False + else: + unknownops.append(opname) + + # get the two heading chars + if len(hds) < 2: + msg = 'Not enough headings (got {0}, need 2), using default -^' + if warnings: + app.warn(msg.format(len(hds)), location) + hds = '-^' + h1, h2 = hds.lstrip()[:2] + + #tell sphinx that the remaining args are invalid. + if len(unknownops) > 0 and app is not None: + opsstrs = ','.join(unknownops) + msg = 'Found additional options ' + opsstrs + ' in automodapi.' + if warnings: + app.warn(msg, location) + + ispkg, hascls, hasfuncs = _mod_info(modnm, toskip) + + #add automodule directive only if no-main-docstr isn't present + if maindocstr: + automodline = '.. automodule:: {modname}'.format(modname=modnm) + else: + automodline = '' + if top_head: + newstrs.append(automod_templ_modheader.format(modname=modnm, + modhds=h1 * len(modnm), + pkgormod='Package' if ispkg else 'Module', + pkgormodhds=h1 * (8 if ispkg else 7), + automoduleline=automodline)) + + if hasfuncs: + newstrs.append(automod_templ_funcs.format(modname=modnm, + funchds=h2 * 9, + toctree=toctreestr, + skips=':skip: ' + ','.join(toskip) if toskip else '')) + + if hascls: + newstrs.append(automod_templ_classes.format(modname=modnm, + clshds=h2 * 7, + toctree=toctreestr, + skips=':skip: ' + ','.join(toskip) if toskip else '')) + + if inhdiag and hascls: + # add inheritance diagram if any classes are in the module + newstrs.append(automod_templ_inh.format( + modname=modnm, clsinhsechds=h2 * 25)) + + newstrs.append(spl[grp * 3 + 3]) + + newsourcestr = ''.join(newstrs) + + if app.config.automodapi_writereprocessed: + #sometimes they are unicode, sometimes not, depending on how sphinx + #has processed things + if isinstance(newsourcestr, unicode): + ustr = newsourcestr + else: + ustr = newsourcestr.decode(app.config.source_encoding) + + if docname is None: + with open(os.path.join(app.srcdir, 'unknown.automodapi'), 'a') as f: + f.write('\n**NEW DOC**\n\n') + f.write(ustr.encode('utf8')) + else: + with open(os.path.join(app.srcdir, docname + '.automodapi'), 'w') as f: + f.write(ustr.encode('utf8')) + + return newsourcestr + else: + return sourcestr + + +def _mod_info(modname, toskip=[]): + """ + Determines if a module is a module or a package and whether or not + it has classes or functions. + """ + + hascls = hasfunc = False + + for localnm, fqnm, obj in zip(*find_mod_objs(modname, onlylocals=True)): + if localnm not in toskip: + hascls = hascls or inspect.isclass(obj) + hasfunc = hasfunc or inspect.isfunction(obj) + if hascls and hasfunc: + break + + #find_mod_objs has already imported modname + pkg = sys.modules[modname] + ispkg = '__init__.' in os.path.split(pkg.__name__)[1] + + return ispkg, hascls, hasfunc + + +def process_automodapi(app, docname, source): + source[0] = automodapi_replace(source[0], app, True, docname) + + +def setup(app): + # need automodsumm for automodapi + app.setup_extension('astropy_helpers.sphinx.ext.automodsumm') + + app.connect('source-read', process_automodapi) + + app.add_config_value('automodapi_toctreedirnm', 'api', True) + app.add_config_value('automodapi_writereprocessed', False, True) diff --git a/astropy_helpers/sphinx/ext/automodsumm.py b/astropy_helpers/sphinx/ext/automodsumm.py new file mode 100644 index 00000000..9c330e30 --- /dev/null +++ b/astropy_helpers/sphinx/ext/automodsumm.py @@ -0,0 +1,550 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This sphinx extension adds two directives for summarizing the public +members of a module or package. + +These directives are primarily for use with the `automodapi` extension, +but can be used independently. + +======================= +`automodsumm` directive +======================= + +This directive will produce an "autosummary"-style table for public +attributes of a specified module. See the `sphinx.ext.autosummary` +extension for details on this process. The main difference from the +`autosummary` directive is that `autosummary` requires manually inputting +all attributes that appear in the table, while this captures the entries +automatically. + +This directive requires a single argument that must be a module or +package. + +It also accepts any options supported by the `autosummary` directive- +see `sphinx.ext.autosummary` for details. It also accepts two additional +options: + + * ``:classes-only:`` + If present, the autosummary table will only contain entries for + classes. This cannot be used at the same time with + ``:functions-only:`` . + + * ``:functions-only:`` + If present, the autosummary table will only contain entries for + functions. This cannot be used at the same time with + ``:classes-only:`` . + + * ``:skip: obj1, [obj2, obj3, ...]`` + If present, specifies that the listed objects should be skipped + and not have their documentation generated, nor be includded in + the summary table. + +This extension also adds one sphinx configuration option: + +* `automodsumm_writereprocessed` + Should be a bool, and if True, will cause `automodsumm` to write files with + any ``automodsumm`` sections replaced with the content Sphinx processes + after ``automodsumm`` has run. The output files are not actually used by + sphinx, so this option is only for figuring out the cause of sphinx warnings + or other debugging. Defaults to `False`. + + +=========================== +`automod-diagram` directive +=========================== + +This directive will produce an inheritance diagram like that of the +`sphinx.ext.inheritance_diagram` extension. + +This directive requires a single argument that must be a module or +package. It accepts no options. + +.. note:: + Like 'inheritance-diagram', 'automod-diagram' requires + `graphviz `_ to generate the inheritance diagram. + +""" + +import inspect +import os +import re + +from sphinx.ext.autosummary import Autosummary +from sphinx.ext.inheritance_diagram import InheritanceDiagram +from docutils.parsers.rst.directives import flag + +from .utils import find_mod_objs +from .astropyautosummary import AstropyAutosummary + + +def _str_list_converter(argument): + """ + A directive option conversion function that converts the option into a list + of strings. Used for 'skip' option. + """ + if argument is None: + return [] + else: + return [s.strip() for s in argument.split(',')] + + +class Automodsumm(AstropyAutosummary): + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + has_content = False + option_spec = dict(Autosummary.option_spec) + option_spec['functions-only'] = flag + option_spec['classes-only'] = flag + option_spec['skip'] = _str_list_converter + + def run(self): + env = self.state.document.settings.env + modname = self.arguments[0] + + self.warnings = [] + nodelist = [] + + try: + localnames, fqns, objs = find_mod_objs(modname) + except ImportError: + self.warnings = [] + self.warn("Couldn't import module " + modname) + return self.warnings + + try: + # set self.content to trick the Autosummary internals. + # Be sure to respect functions-only and classes-only. + funconly = 'functions-only' in self.options + clsonly = 'classes-only' in self.options + + skipnames = [] + if 'skip' in self.options: + option_skipnames = set(self.options['skip']) + for lnm in localnames: + if lnm in option_skipnames: + option_skipnames.remove(lnm) + skipnames.append(lnm) + if len(option_skipnames) > 0: + self.warn('Tried to skip objects {objs} in module {mod}, ' + 'but they were not present. Ignoring.'.format( + objs=option_skipnames, mod=modname)) + + if funconly and not clsonly: + cont = [] + for nm, obj in zip(localnames, objs): + if nm not in skipnames and inspect.isfunction(obj): + cont.append(nm) + elif clsonly: + cont = [] + for nm, obj in zip(localnames, objs): + if nm not in skipnames and inspect.isclass(obj): + cont.append(nm) + else: + if clsonly and funconly: + self.warning('functions-only and classes-only both ' + 'defined. Skipping.') + cont = [nm for nm in localnames if nm not in skipnames] + + self.content = cont + + #for some reason, even though ``currentmodule`` is substituted in, sphinx + #doesn't necessarily recognize this fact. So we just force it + #internally, and that seems to fix things + env.temp_data['py:module'] = modname + + #can't use super because Sphinx/docutils has trouble + #return super(Autosummary,self).run() + nodelist.extend(Autosummary.run(self)) + return self.warnings + nodelist + finally: # has_content = False for the Automodsumm + self.content = [] + + +#<-------------------automod-diagram stuff------------------------------------> +class Automoddiagram(InheritanceDiagram): + def run(self): + try: + nms, objs = find_mod_objs(self.arguments[0], onlylocals=True)[1:] + except ImportError: + self.warnings = [] + self.warn("Couldn't import module " + self.arguments[0]) + return self.warnings + + clsnms = [] + for n, o in zip(nms, objs): + + if inspect.isclass(o): + clsnms.append(n) + + oldargs = self.arguments + try: + if len(clsnms) > 0: + self.arguments = [u' '.join(clsnms)] + return InheritanceDiagram.run(self) + finally: + self.arguments = oldargs + + +#<---------------------automodsumm generation stuff---------------------------> +def process_automodsumm_generation(app): + env = app.builder.env + ext = app.config.source_suffix + + filestosearch = [x + ext for x in env.found_docs + if os.path.isfile(env.doc2path(x))]\ + + liness = [] + for sfn in filestosearch: + lines = automodsumm_to_autosummary_lines(sfn, app) + liness.append(lines) + if app.config.automodsumm_writereprocessed: + if lines: # empty list means no automodsumm entry is in the file + outfn = os.path.join(app.srcdir, sfn) + '.automodsumm' + with open(outfn, 'w') as f: + for l in lines: + f.write(l) + f.write('\n') + + for sfn, lines in zip(filestosearch, liness): + if len(lines) > 0: + generate_automodsumm_docs(lines, sfn, builder=app.builder, + warn=app.warn, info=app.info, + suffix=app.config.source_suffix, + base_path=app.srcdir) + +#_automodsummrex = re.compile(r'^(\s*)\.\. automodsumm::\s*([A-Za-z0-9_.]+)\s*' +# r'\n\1(\s*)(\S|$)', re.MULTILINE) +_lineendrex = r'(?:\n|$)' +_hdrex = r'^\n?(\s*)\.\. automodsumm::\s*(\S+)\s*' + _lineendrex +_oprex1 = r'(?:\1(\s+)\S.*' + _lineendrex + ')' +_oprex2 = r'(?:\1\4\S.*' + _lineendrex + ')' +_automodsummrex = re.compile(_hdrex + '(' + _oprex1 + '?' + _oprex2 + '*)', + re.MULTILINE) + + +def automodsumm_to_autosummary_lines(fn, app): + """ + Generates lines from a file with an "automodsumm" entry suitable for + feeding into "autosummary". + + Searches the provided file for `automodsumm` directives and returns + a list of lines specifying the `autosummary` commands for the modules + requested. This does *not* return the whole file contents - just an + autosummary section in place of any :automodsumm: entries. Note that + any options given for `automodsumm` are also included in the + generated `autosummary` section. + + Parameters + ---------- + fn : str + The name of the file to search for `automodsumm` entries. + app : sphinx.application.Application + The sphinx Application object + + Return + ------ + lines : list of str + Lines for all `automodsumm` entries with the entries replaced by + `autosummary` and the module's members added. + + + """ + fullfn = os.path.join(app.builder.env.srcdir, fn) + + with open(fullfn) as fr: + if 'astropy_helpers.sphinx.ext.automodapi' in app._extensions: + from astropy_helpers.sphinx.ext.automodapi import automodapi_replace + # Must do the automodapi on the source to get the automodsumm + # that might be in there + filestr = automodapi_replace(fr.read(), app, True, fn, False) + else: + filestr = fr.read() + + spl = _automodsummrex.split(filestr) + #0th entry is the stuff before the first automodsumm line + indent1s = spl[1::5] + mods = spl[2::5] + opssecs = spl[3::5] + indent2s = spl[4::5] + remainders = spl[5::5] + + # only grab automodsumm sections and convert them to autosummary with the + # entries for all the public objects + newlines = [] + + #loop over all automodsumms in this document + for i, (i1, i2, modnm, ops, rem) in enumerate(zip(indent1s, indent2s, mods, + opssecs, remainders)): + allindent = i1 + i2 + + #filter out functions-only and classes-only options if present + oplines = ops.split('\n') + toskip = [] + funcsonly = clssonly = False + for i, ln in reversed(list(enumerate(oplines))): + if ':functions-only:' in ln: + funcsonly = True + del oplines[i] + if ':classes-only:' in ln: + clssonly = True + del oplines[i] + if ':skip:' in ln: + toskip.extend(_str_list_converter(ln.replace(':skip:', ''))) + del oplines[i] + if funcsonly and clssonly: + msg = ('Defined both functions-only and classes-only options. ' + 'Skipping this directive.') + lnnum = sum([spl[j].count('\n') for j in range(i * 5 + 1)]) + app.warn('[automodsumm]' + msg, (fn, lnnum)) + continue + + # Use the currentmodule directive so we can just put the local names + # in the autosummary table. Note that this doesn't always seem to + # actually "take" in Sphinx's eyes, so in `Automodsumm.run`, we have to + # force it internally, as well. + newlines.extend([i1 + '.. currentmodule:: ' + modnm, + '', + '.. autosummary::']) + newlines.extend(oplines) + + for nm, fqn, obj in zip(*find_mod_objs(modnm, onlylocals=True)): + if nm in toskip: + continue + if funcsonly and not inspect.isfunction(obj): + continue + if clssonly and not inspect.isclass(obj): + continue + newlines.append(allindent + nm) + + return newlines + + +def generate_automodsumm_docs(lines, srcfn, suffix='.rst', warn=None, + info=None, base_path=None, builder=None, + template_dir=None): + """ + This function is adapted from + `sphinx.ext.autosummary.generate.generate_autosummmary_docs` to + generate source for the automodsumm directives that should be + autosummarized. Unlike generate_autosummary_docs, this function is + called one file at a time. + """ + + from sphinx.jinja2glue import BuiltinTemplateLoader + from sphinx.ext.autosummary import import_by_name, get_documenter + from sphinx.ext.autosummary.generate import (find_autosummary_in_lines, + _simple_info, _simple_warn) + from sphinx.util.osutil import ensuredir + from sphinx.util.inspect import safe_getattr + from jinja2 import FileSystemLoader, TemplateNotFound + from jinja2.sandbox import SandboxedEnvironment + + if info is None: + info = _simple_info + if warn is None: + warn = _simple_warn + + #info('[automodsumm] generating automodsumm for: ' + srcfn) + + # Create our own templating environment - here we use Astropy's + # templates rather than the default autosummary templates, in order to + # allow docstrings to be shown for methods. + template_dirs = [os.path.join(os.path.dirname(__file__), 'templates'), + os.path.join(base_path, '_templates')] + if builder is not None: + # allow the user to override the templates + template_loader = BuiltinTemplateLoader() + template_loader.init(builder, dirs=template_dirs) + else: + if template_dir: + template_dirs.insert(0, template_dir) + template_loader = FileSystemLoader(template_dirs) + template_env = SandboxedEnvironment(loader=template_loader) + + # read + #items = find_autosummary_in_files(sources) + items = find_autosummary_in_lines(lines, filename=srcfn) + if len(items) > 0: + msg = '[automodsumm] {1}: found {0} automodsumm entries to generate' + info(msg.format(len(items), srcfn)) + +# gennms = [item[0] for item in items] +# if len(gennms) > 20: +# gennms = gennms[:10] + ['...'] + gennms[-10:] +# info('[automodsumm] generating autosummary for: ' + ', '.join(gennms)) + + # remove possible duplicates + items = dict([(item, True) for item in items]).keys() + + # keep track of new files + new_files = [] + + # write + for name, path, template_name in sorted(items): + if path is None: + # The corresponding autosummary:: directive did not have + # a :toctree: option + continue + + path = os.path.abspath(path) + ensuredir(path) + + try: + name, obj, parent = import_by_name(name) + except ImportError, e: + warn('[automodsumm] failed to import %r: %s' % (name, e)) + continue + + fn = os.path.join(path, name + suffix) + + # skip it if it exists + if os.path.isfile(fn): + continue + + new_files.append(fn) + + f = open(fn, 'w') + + try: + doc = get_documenter(obj, parent) + + if template_name is not None: + template = template_env.get_template(template_name) + else: + tmplstr = 'autosummary/%s.rst' + try: + template = template_env.get_template(tmplstr % doc.objtype) + except TemplateNotFound: + template = template_env.get_template(tmplstr % 'base') + + def get_members_mod(obj, typ, include_public=[]): + """ + typ = None -> all + """ + items = [] + for name in dir(obj): + try: + documenter = get_documenter(safe_getattr(obj, name), + obj) + except AttributeError: + continue + if typ is None or documenter.objtype == typ: + items.append(name) + public = [x for x in items + if x in include_public or not x.startswith('_')] + return public, items + + def get_members_class(obj, typ, include_public=[], + include_base=False): + """ + typ = None -> all + include_base -> include attrs that are from a base class + """ + items = [] + + # using dir gets all of the attributes, including the elements + # from the base class, otherwise use __slots__ or __dict__ + if include_base: + names = dir(obj) + else: + if hasattr(obj, '__slots__'): + names = tuple(getattr(obj, '__slots__')) + else: + names = getattr(obj, '__dict__').keys() + + for name in names: + try: + documenter = get_documenter(safe_getattr(obj, name), + obj) + except AttributeError: + continue + if typ is None or documenter.objtype == typ: + items.append(name) + public = [x for x in items + if x in include_public or not x.startswith('_')] + return public, items + + ns = {} + + if doc.objtype == 'module': + ns['members'] = get_members_mod(obj, None) + ns['functions'], ns['all_functions'] = \ + get_members_mod(obj, 'function') + ns['classes'], ns['all_classes'] = \ + get_members_mod(obj, 'class') + ns['exceptions'], ns['all_exceptions'] = \ + get_members_mod(obj, 'exception') + elif doc.objtype == 'class': + api_class_methods = ['__init__', '__call__'] + ns['members'] = get_members_class(obj, None) + ns['methods'], ns['all_methods'] = \ + get_members_class(obj, 'method', api_class_methods) + ns['attributes'], ns['all_attributes'] = \ + get_members_class(obj, 'attribute') + ns['methods'].sort() + ns['attributes'].sort() + + parts = name.split('.') + if doc.objtype in ('method', 'attribute'): + mod_name = '.'.join(parts[:-2]) + cls_name = parts[-2] + obj_name = '.'.join(parts[-2:]) + ns['class'] = cls_name + else: + mod_name, obj_name = '.'.join(parts[:-1]), parts[-1] + + ns['fullname'] = name + ns['module'] = mod_name + ns['objname'] = obj_name + ns['name'] = parts[-1] + + ns['objtype'] = doc.objtype + ns['underline'] = len(name) * '=' + + # We now check whether a file for reference footnotes exists for + # the module being documented. We first check if the + # current module is a file or a directory, as this will give a + # different path for the reference file. For example, if + # documenting astropy.wcs then the reference file is at + # ../wcs/references.txt, while if we are documenting + # astropy.config.logging_helper (which is at + # astropy/config/logging_helper.py) then the reference file is set + # to ../config/references.txt + if '.' in mod_name: + mod_name_dir = mod_name.replace('.', '/').split('/', 1)[1] + else: + mod_name_dir = mod_name + if not os.path.isdir(os.path.join(base_path, mod_name_dir)) \ + and os.path.isdir(os.path.join(base_path, mod_name_dir.rsplit('/', 1)[0])): + mod_name_dir = mod_name_dir.rsplit('/', 1)[0] + + # We then have to check whether it exists, and if so, we pass it + # to the template. + if os.path.exists(os.path.join(base_path, mod_name_dir, 'references.txt')): + # An important subtlety here is that the path we pass in has + # to be relative to the file being generated, so we have to + # figure out the right number of '..'s + ndirsback = path.replace(base_path, '').count('/') + ref_file_rel_segments = ['..'] * ndirsback + ref_file_rel_segments.append(mod_name_dir) + ref_file_rel_segments.append('references.txt') + ns['referencefile'] = os.path.join(*ref_file_rel_segments) + + rendered = template.render(**ns) + f.write(rendered) + finally: + f.close() + + +def setup(app): + # need our autosummary + app.setup_extension('astropy_helpers.sphinx.ext.astropyautosummary') + # need inheritance-diagram for automod-diagram + app.setup_extension('sphinx.ext.inheritance_diagram') + + app.add_directive('automod-diagram', Automoddiagram) + app.add_directive('automodsumm', Automodsumm) + app.connect('builder-inited', process_automodsumm_generation) + + app.add_config_value('automodsumm_writereprocessed', False, True) diff --git a/astropy_helpers/sphinx/ext/changelog_links.py b/astropy_helpers/sphinx/ext/changelog_links.py new file mode 100644 index 00000000..44085478 --- /dev/null +++ b/astropy_helpers/sphinx/ext/changelog_links.py @@ -0,0 +1,66 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This sphinx extension makes the issue numbers in the changelog into links to +GitHub issues. +""" + +from __future__ import print_function + +import re + +from docutils.nodes import Text, reference + +BLOCK_PATTERN = re.compile('\[#.+\]', flags=re.DOTALL) +ISSUE_PATTERN = re.compile('#[0-9]+') + + +def process_changelog_links(app, doctree, docname): + + if 'changelog' not in docname or app.config.github_issues_url is None: + return + + for item in doctree.traverse(): + + if not isinstance(item, Text): + continue + + # We build a new list of items to replace the current item. If + # a link is found, we need to use a 'reference' item. + children = [] + + # First cycle through blocks of issues (delimited by []) then + # iterate inside each one to find the individual issues. + prev_block_end = 0 + for block in BLOCK_PATTERN.finditer(item): + block_start, block_end = block.start(), block.end() + children.append(Text(item[prev_block_end:block_start])) + block = item[block_start:block_end] + prev_end = 0 + for m in ISSUE_PATTERN.finditer(block): + start, end = m.start(), m.end() + children.append(Text(block[prev_end:start])) + issue_number = block[start:end] + refuri = app.config.github_issues_url + issue_number[1:] + children.append(reference(text=issue_number, + name=issue_number, + refuri=refuri)) + prev_end = end + + prev_block_end = block_end + + # If no issues were found, this adds the whole item, + # otherwise it adds the remaining text. + children.append(Text(block[prev_end:block_end])) + + # If no blocks were found, this adds the whole item, otherwise + # it adds the remaining text. + children.append(Text(item[prev_block_end:])) + + # Replace item by the new list of items we have generated, + # which may contain links. + item.parent.replace(item, children) + + +def setup(app): + app.connect('doctree-resolved', process_changelog_links) + app.add_config_value('github_issues_url', None, True) diff --git a/astropy_helpers/sphinx/ext/comment_eater.py b/astropy_helpers/sphinx/ext/comment_eater.py new file mode 100644 index 00000000..e11eea90 --- /dev/null +++ b/astropy_helpers/sphinx/ext/comment_eater.py @@ -0,0 +1,158 @@ +from cStringIO import StringIO +import compiler +import inspect +import textwrap +import tokenize + +from compiler_unparse import unparse + + +class Comment(object): + """ A comment block. + """ + is_comment = True + def __init__(self, start_lineno, end_lineno, text): + # int : The first line number in the block. 1-indexed. + self.start_lineno = start_lineno + # int : The last line number. Inclusive! + self.end_lineno = end_lineno + # str : The text block including '#' character but not any leading spaces. + self.text = text + + def add(self, string, start, end, line): + """ Add a new comment line. + """ + self.start_lineno = min(self.start_lineno, start[0]) + self.end_lineno = max(self.end_lineno, end[0]) + self.text += string + + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.start_lineno, + self.end_lineno, self.text) + + +class NonComment(object): + """ A non-comment block of code. + """ + is_comment = False + def __init__(self, start_lineno, end_lineno): + self.start_lineno = start_lineno + self.end_lineno = end_lineno + + def add(self, string, start, end, line): + """ Add lines to the block. + """ + if string.strip(): + # Only add if not entirely whitespace. + self.start_lineno = min(self.start_lineno, start[0]) + self.end_lineno = max(self.end_lineno, end[0]) + + def __repr__(self): + return '%s(%r, %r)' % (self.__class__.__name__, self.start_lineno, + self.end_lineno) + + +class CommentBlocker(object): + """ Pull out contiguous comment blocks. + """ + def __init__(self): + # Start with a dummy. + self.current_block = NonComment(0, 0) + + # All of the blocks seen so far. + self.blocks = [] + + # The index mapping lines of code to their associated comment blocks. + self.index = {} + + def process_file(self, file): + """ Process a file object. + """ + for token in tokenize.generate_tokens(file.next): + self.process_token(*token) + self.make_index() + + def process_token(self, kind, string, start, end, line): + """ Process a single token. + """ + if self.current_block.is_comment: + if kind == tokenize.COMMENT: + self.current_block.add(string, start, end, line) + else: + self.new_noncomment(start[0], end[0]) + else: + if kind == tokenize.COMMENT: + self.new_comment(string, start, end, line) + else: + self.current_block.add(string, start, end, line) + + def new_noncomment(self, start_lineno, end_lineno): + """ We are transitioning from a noncomment to a comment. + """ + block = NonComment(start_lineno, end_lineno) + self.blocks.append(block) + self.current_block = block + + def new_comment(self, string, start, end, line): + """ Possibly add a new comment. + + Only adds a new comment if this comment is the only thing on the line. + Otherwise, it extends the noncomment block. + """ + prefix = line[:start[1]] + if prefix.strip(): + # Oops! Trailing comment, not a comment block. + self.current_block.add(string, start, end, line) + else: + # A comment block. + block = Comment(start[0], end[0], string) + self.blocks.append(block) + self.current_block = block + + def make_index(self): + """ Make the index mapping lines of actual code to their associated + prefix comments. + """ + for prev, block in zip(self.blocks[:-1], self.blocks[1:]): + if not block.is_comment: + self.index[block.start_lineno] = prev + + def search_for_comment(self, lineno, default=None): + """ Find the comment block just before the given line number. + + Returns None (or the specified default) if there is no such block. + """ + if not self.index: + self.make_index() + block = self.index.get(lineno, None) + text = getattr(block, 'text', default) + return text + + +def strip_comment_marker(text): + """ Strip # markers at the front of a block of comment text. + """ + lines = [] + for line in text.splitlines(): + lines.append(line.lstrip('#')) + text = textwrap.dedent('\n'.join(lines)) + return text + + +def get_class_traits(klass): + """ Yield all of the documentation for trait definitions on a class object. + """ + # FIXME: gracefully handle errors here or in the caller? + source = inspect.getsource(klass) + cb = CommentBlocker() + cb.process_file(StringIO(source)) + mod_ast = compiler.parse(source) + class_ast = mod_ast.node.nodes[0] + for node in class_ast.code.nodes: + # FIXME: handle other kinds of assignments? + if isinstance(node, compiler.ast.Assign): + name = node.nodes[0].name + rhs = unparse(node.expr).strip() + doc = strip_comment_marker(cb.search_for_comment(node.lineno, default='')) + yield name, rhs, doc + diff --git a/astropy_helpers/sphinx/ext/compiler_unparse.py b/astropy_helpers/sphinx/ext/compiler_unparse.py new file mode 100644 index 00000000..ffcf51b3 --- /dev/null +++ b/astropy_helpers/sphinx/ext/compiler_unparse.py @@ -0,0 +1,860 @@ +""" Turn compiler.ast structures back into executable python code. + + The unparse method takes a compiler.ast tree and transforms it back into + valid python code. It is incomplete and currently only works for + import statements, function calls, function definitions, assignments, and + basic expressions. + + Inspired by python-2.5-svn/Demo/parser/unparse.py + + fixme: We may want to move to using _ast trees because the compiler for + them is about 6 times faster than compiler.compile. +""" + +import sys +import cStringIO +from compiler.ast import Const, Name, Tuple, Div, Mul, Sub, Add + +def unparse(ast, single_line_functions=False): + s = cStringIO.StringIO() + UnparseCompilerAst(ast, s, single_line_functions) + return s.getvalue().lstrip() + +op_precedence = { 'compiler.ast.Power':3, 'compiler.ast.Mul':2, 'compiler.ast.Div':2, + 'compiler.ast.Add':1, 'compiler.ast.Sub':1 } + +class UnparseCompilerAst: + """ Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarged. + """ + + ######################################################################### + # object interface. + ######################################################################### + + def __init__(self, tree, file = sys.stdout, single_line_functions=False): + """ Unparser(tree, file=sys.stdout) -> None. + + Print the source for tree to file. + """ + self.f = file + self._single_func = single_line_functions + self._do_indent = True + self._indent = 0 + self._dispatch(tree) + self._write("\n") + self.f.flush() + + ######################################################################### + # Unparser private interface. + ######################################################################### + + ### format, output, and dispatch methods ################################ + + def _fill(self, text = ""): + "Indent a piece of text, according to the current indentation level" + if self._do_indent: + self._write("\n"+" "*self._indent + text) + else: + self._write(text) + + def _write(self, text): + "Append a piece of text to the current line." + self.f.write(text) + + def _enter(self): + "Print ':', and increase the indentation." + self._write(": ") + self._indent += 1 + + def _leave(self): + "Decrease the indentation level." + self._indent -= 1 + + def _dispatch(self, tree): + "_dispatcher function, _dispatching tree type T to method _T." + if isinstance(tree, list): + for t in tree: + self._dispatch(t) + return + meth = getattr(self, "_"+tree.__class__.__name__) + if tree.__class__.__name__ == 'NoneType' and not self._do_indent: + return + meth(tree) + + + ######################################################################### + # compiler.ast unparsing methods. + # + # There should be one method per concrete grammar type. They are + # organized in alphabetical order. + ######################################################################### + + def _Add(self, t): + self.__binary_op(t, '+') + + def _And(self, t): + self._write(" (") + for i, node in enumerate(t.nodes): + self._dispatch(node) + if i != len(t.nodes)-1: + self._write(") and (") + self._write(")") + + def _AssAttr(self, t): + """ Handle assigning an attribute of an object + """ + self._dispatch(t.expr) + self._write('.'+t.attrname) + + def _Assign(self, t): + """ Expression Assignment such as "a = 1". + + This only handles assignment in expressions. Keyword assignment + is handled separately. + """ + self._fill() + for target in t.nodes: + self._dispatch(target) + self._write(" = ") + self._dispatch(t.expr) + if not self._do_indent: + self._write('; ') + + def _AssName(self, t): + """ Name on left hand side of expression. + + Treat just like a name on the right side of an expression. + """ + self._Name(t) + + def _AssTuple(self, t): + """ Tuple on left hand side of an expression. + """ + + # _write each elements, separated by a comma. + for element in t.nodes[:-1]: + self._dispatch(element) + self._write(", ") + + # Handle the last one without writing comma + last_element = t.nodes[-1] + self._dispatch(last_element) + + def _AugAssign(self, t): + """ +=,-=,*=,/=,**=, etc. operations + """ + + self._fill() + self._dispatch(t.node) + self._write(' '+t.op+' ') + self._dispatch(t.expr) + if not self._do_indent: + self._write(';') + + def _Bitand(self, t): + """ Bit and operation. + """ + + for i, node in enumerate(t.nodes): + self._write("(") + self._dispatch(node) + self._write(")") + if i != len(t.nodes)-1: + self._write(" & ") + + def _Bitor(self, t): + """ Bit or operation + """ + + for i, node in enumerate(t.nodes): + self._write("(") + self._dispatch(node) + self._write(")") + if i != len(t.nodes)-1: + self._write(" | ") + + def _CallFunc(self, t): + """ Function call. + """ + self._dispatch(t.node) + self._write("(") + comma = False + for e in t.args: + if comma: self._write(", ") + else: comma = True + self._dispatch(e) + if t.star_args: + if comma: self._write(", ") + else: comma = True + self._write("*") + self._dispatch(t.star_args) + if t.dstar_args: + if comma: self._write(", ") + else: comma = True + self._write("**") + self._dispatch(t.dstar_args) + self._write(")") + + def _Compare(self, t): + self._dispatch(t.expr) + for op, expr in t.ops: + self._write(" " + op + " ") + self._dispatch(expr) + + def _Const(self, t): + """ A constant value such as an integer value, 3, or a string, "hello". + """ + self._dispatch(t.value) + + def _Decorators(self, t): + """ Handle function decorators (eg. @has_units) + """ + for node in t.nodes: + self._dispatch(node) + + def _Dict(self, t): + self._write("{") + for i, (k, v) in enumerate(t.items): + self._dispatch(k) + self._write(": ") + self._dispatch(v) + if i < len(t.items)-1: + self._write(", ") + self._write("}") + + def _Discard(self, t): + """ Node for when return value is ignored such as in "foo(a)". + """ + self._fill() + self._dispatch(t.expr) + + def _Div(self, t): + self.__binary_op(t, '/') + + def _Ellipsis(self, t): + self._write("...") + + def _From(self, t): + """ Handle "from xyz import foo, bar as baz". + """ + # fixme: Are From and ImportFrom handled differently? + self._fill("from ") + self._write(t.modname) + self._write(" import ") + for i, (name,asname) in enumerate(t.names): + if i != 0: + self._write(", ") + self._write(name) + if asname is not None: + self._write(" as "+asname) + + def _Function(self, t): + """ Handle function definitions + """ + if t.decorators is not None: + self._fill("@") + self._dispatch(t.decorators) + self._fill("def "+t.name + "(") + defaults = [None] * (len(t.argnames) - len(t.defaults)) + list(t.defaults) + for i, arg in enumerate(zip(t.argnames, defaults)): + self._write(arg[0]) + if arg[1] is not None: + self._write('=') + self._dispatch(arg[1]) + if i < len(t.argnames)-1: + self._write(', ') + self._write(")") + if self._single_func: + self._do_indent = False + self._enter() + self._dispatch(t.code) + self._leave() + self._do_indent = True + + def _Getattr(self, t): + """ Handle getting an attribute of an object + """ + if isinstance(t.expr, (Div, Mul, Sub, Add)): + self._write('(') + self._dispatch(t.expr) + self._write(')') + else: + self._dispatch(t.expr) + + self._write('.'+t.attrname) + + def _If(self, t): + self._fill() + + for i, (compare,code) in enumerate(t.tests): + if i == 0: + self._write("if ") + else: + self._write("elif ") + self._dispatch(compare) + self._enter() + self._fill() + self._dispatch(code) + self._leave() + self._write("\n") + + if t.else_ is not None: + self._write("else") + self._enter() + self._fill() + self._dispatch(t.else_) + self._leave() + self._write("\n") + + def _IfExp(self, t): + self._dispatch(t.then) + self._write(" if ") + self._dispatch(t.test) + + if t.else_ is not None: + self._write(" else (") + self._dispatch(t.else_) + self._write(")") + + def _Import(self, t): + """ Handle "import xyz.foo". + """ + self._fill("import ") + + for i, (name,asname) in enumerate(t.names): + if i != 0: + self._write(", ") + self._write(name) + if asname is not None: + self._write(" as "+asname) + + def _Keyword(self, t): + """ Keyword value assignment within function calls and definitions. + """ + self._write(t.name) + self._write("=") + self._dispatch(t.expr) + + def _List(self, t): + self._write("[") + for i,node in enumerate(t.nodes): + self._dispatch(node) + if i < len(t.nodes)-1: + self._write(", ") + self._write("]") + + def _Module(self, t): + if t.doc is not None: + self._dispatch(t.doc) + self._dispatch(t.node) + + def _Mul(self, t): + self.__binary_op(t, '*') + + def _Name(self, t): + self._write(t.name) + + def _NoneType(self, t): + self._write("None") + + def _Not(self, t): + self._write('not (') + self._dispatch(t.expr) + self._write(')') + + def _Or(self, t): + self._write(" (") + for i, node in enumerate(t.nodes): + self._dispatch(node) + if i != len(t.nodes)-1: + self._write(") or (") + self._write(")") + + def _Pass(self, t): + self._write("pass\n") + + def _Printnl(self, t): + self._fill("print ") + if t.dest: + self._write(">> ") + self._dispatch(t.dest) + self._write(", ") + comma = False + for node in t.nodes: + if comma: self._write(', ') + else: comma = True + self._dispatch(node) + + def _Power(self, t): + self.__binary_op(t, '**') + + def _Return(self, t): + self._fill("return ") + if t.value: + if isinstance(t.value, Tuple): + text = ', '.join([ name.name for name in t.value.asList() ]) + self._write(text) + else: + self._dispatch(t.value) + if not self._do_indent: + self._write('; ') + + def _Slice(self, t): + self._dispatch(t.expr) + self._write("[") + if t.lower: + self._dispatch(t.lower) + self._write(":") + if t.upper: + self._dispatch(t.upper) + #if t.step: + # self._write(":") + # self._dispatch(t.step) + self._write("]") + + def _Sliceobj(self, t): + for i, node in enumerate(t.nodes): + if i != 0: + self._write(":") + if not (isinstance(node, Const) and node.value is None): + self._dispatch(node) + + def _Stmt(self, tree): + for node in tree.nodes: + self._dispatch(node) + + def _Sub(self, t): + self.__binary_op(t, '-') + + def _Subscript(self, t): + self._dispatch(t.expr) + self._write("[") + for i, value in enumerate(t.subs): + if i != 0: + self._write(",") + self._dispatch(value) + self._write("]") + + def _TryExcept(self, t): + self._fill("try") + self._enter() + self._dispatch(t.body) + self._leave() + + for handler in t.handlers: + self._fill('except ') + self._dispatch(handler[0]) + if handler[1] is not None: + self._write(', ') + self._dispatch(handler[1]) + self._enter() + self._dispatch(handler[2]) + self._leave() + + if t.else_: + self._fill("else") + self._enter() + self._dispatch(t.else_) + self._leave() + + def _Tuple(self, t): + + if not t.nodes: + # Empty tuple. + self._write("()") + else: + self._write("(") + + # _write each elements, separated by a comma. + for element in t.nodes[:-1]: + self._dispatch(element) + self._write(", ") + + # Handle the last one without writing comma + last_element = t.nodes[-1] + self._dispatch(last_element) + + self._write(")") + + def _UnaryAdd(self, t): + self._write("+") + self._dispatch(t.expr) + + def _UnarySub(self, t): + self._write("-") + self._dispatch(t.expr) + + def _With(self, t): + self._fill('with ') + self._dispatch(t.expr) + if t.vars: + self._write(' as ') + self._dispatch(t.vars.name) + self._enter() + self._dispatch(t.body) + self._leave() + self._write('\n') + + def _int(self, t): + self._write(repr(t)) + + def __binary_op(self, t, symbol): + # Check if parenthesis are needed on left side and then dispatch + has_paren = False + left_class = str(t.left.__class__) + if (left_class in op_precedence.keys() and + op_precedence[left_class] < op_precedence[str(t.__class__)]): + has_paren = True + if has_paren: + self._write('(') + self._dispatch(t.left) + if has_paren: + self._write(')') + # Write the appropriate symbol for operator + self._write(symbol) + # Check if parenthesis are needed on the right side and then dispatch + has_paren = False + right_class = str(t.right.__class__) + if (right_class in op_precedence.keys() and + op_precedence[right_class] < op_precedence[str(t.__class__)]): + has_paren = True + if has_paren: + self._write('(') + self._dispatch(t.right) + if has_paren: + self._write(')') + + def _float(self, t): + # if t is 0.1, str(t)->'0.1' while repr(t)->'0.1000000000001' + # We prefer str here. + self._write(str(t)) + + def _str(self, t): + self._write(repr(t)) + + def _tuple(self, t): + self._write(str(t)) + + ######################################################################### + # These are the methods from the _ast modules unparse. + # + # As our needs to handle more advanced code increase, we may want to + # modify some of the methods below so that they work for compiler.ast. + ######################################################################### + +# # stmt +# def _Expr(self, tree): +# self._fill() +# self._dispatch(tree.value) +# +# def _Import(self, t): +# self._fill("import ") +# first = True +# for a in t.names: +# if first: +# first = False +# else: +# self._write(", ") +# self._write(a.name) +# if a.asname: +# self._write(" as "+a.asname) +# +## def _ImportFrom(self, t): +## self._fill("from ") +## self._write(t.module) +## self._write(" import ") +## for i, a in enumerate(t.names): +## if i == 0: +## self._write(", ") +## self._write(a.name) +## if a.asname: +## self._write(" as "+a.asname) +## # XXX(jpe) what is level for? +## +# +# def _Break(self, t): +# self._fill("break") +# +# def _Continue(self, t): +# self._fill("continue") +# +# def _Delete(self, t): +# self._fill("del ") +# self._dispatch(t.targets) +# +# def _Assert(self, t): +# self._fill("assert ") +# self._dispatch(t.test) +# if t.msg: +# self._write(", ") +# self._dispatch(t.msg) +# +# def _Exec(self, t): +# self._fill("exec ") +# self._dispatch(t.body) +# if t.globals: +# self._write(" in ") +# self._dispatch(t.globals) +# if t.locals: +# self._write(", ") +# self._dispatch(t.locals) +# +# def _Print(self, t): +# self._fill("print ") +# do_comma = False +# if t.dest: +# self._write(">>") +# self._dispatch(t.dest) +# do_comma = True +# for e in t.values: +# if do_comma:self._write(", ") +# else:do_comma=True +# self._dispatch(e) +# if not t.nl: +# self._write(",") +# +# def _Global(self, t): +# self._fill("global") +# for i, n in enumerate(t.names): +# if i != 0: +# self._write(",") +# self._write(" " + n) +# +# def _Yield(self, t): +# self._fill("yield") +# if t.value: +# self._write(" (") +# self._dispatch(t.value) +# self._write(")") +# +# def _Raise(self, t): +# self._fill('raise ') +# if t.type: +# self._dispatch(t.type) +# if t.inst: +# self._write(", ") +# self._dispatch(t.inst) +# if t.tback: +# self._write(", ") +# self._dispatch(t.tback) +# +# +# def _TryFinally(self, t): +# self._fill("try") +# self._enter() +# self._dispatch(t.body) +# self._leave() +# +# self._fill("finally") +# self._enter() +# self._dispatch(t.finalbody) +# self._leave() +# +# def _excepthandler(self, t): +# self._fill("except ") +# if t.type: +# self._dispatch(t.type) +# if t.name: +# self._write(", ") +# self._dispatch(t.name) +# self._enter() +# self._dispatch(t.body) +# self._leave() +# +# def _ClassDef(self, t): +# self._write("\n") +# self._fill("class "+t.name) +# if t.bases: +# self._write("(") +# for a in t.bases: +# self._dispatch(a) +# self._write(", ") +# self._write(")") +# self._enter() +# self._dispatch(t.body) +# self._leave() +# +# def _FunctionDef(self, t): +# self._write("\n") +# for deco in t.decorators: +# self._fill("@") +# self._dispatch(deco) +# self._fill("def "+t.name + "(") +# self._dispatch(t.args) +# self._write(")") +# self._enter() +# self._dispatch(t.body) +# self._leave() +# +# def _For(self, t): +# self._fill("for ") +# self._dispatch(t.target) +# self._write(" in ") +# self._dispatch(t.iter) +# self._enter() +# self._dispatch(t.body) +# self._leave() +# if t.orelse: +# self._fill("else") +# self._enter() +# self._dispatch(t.orelse) +# self._leave +# +# def _While(self, t): +# self._fill("while ") +# self._dispatch(t.test) +# self._enter() +# self._dispatch(t.body) +# self._leave() +# if t.orelse: +# self._fill("else") +# self._enter() +# self._dispatch(t.orelse) +# self._leave +# +# # expr +# def _Str(self, tree): +# self._write(repr(tree.s)) +## +# def _Repr(self, t): +# self._write("`") +# self._dispatch(t.value) +# self._write("`") +# +# def _Num(self, t): +# self._write(repr(t.n)) +# +# def _ListComp(self, t): +# self._write("[") +# self._dispatch(t.elt) +# for gen in t.generators: +# self._dispatch(gen) +# self._write("]") +# +# def _GeneratorExp(self, t): +# self._write("(") +# self._dispatch(t.elt) +# for gen in t.generators: +# self._dispatch(gen) +# self._write(")") +# +# def _comprehension(self, t): +# self._write(" for ") +# self._dispatch(t.target) +# self._write(" in ") +# self._dispatch(t.iter) +# for if_clause in t.ifs: +# self._write(" if ") +# self._dispatch(if_clause) +# +# def _IfExp(self, t): +# self._dispatch(t.body) +# self._write(" if ") +# self._dispatch(t.test) +# if t.orelse: +# self._write(" else ") +# self._dispatch(t.orelse) +# +# unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} +# def _UnaryOp(self, t): +# self._write(self.unop[t.op.__class__.__name__]) +# self._write("(") +# self._dispatch(t.operand) +# self._write(")") +# +# binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", +# "LShift":">>", "RShift":"<<", "BitOr":"|", "BitXor":"^", "BitAnd":"&", +# "FloorDiv":"//", "Pow": "**"} +# def _BinOp(self, t): +# self._write("(") +# self._dispatch(t.left) +# self._write(")" + self.binop[t.op.__class__.__name__] + "(") +# self._dispatch(t.right) +# self._write(")") +# +# boolops = {_ast.And: 'and', _ast.Or: 'or'} +# def _BoolOp(self, t): +# self._write("(") +# self._dispatch(t.values[0]) +# for v in t.values[1:]: +# self._write(" %s " % self.boolops[t.op.__class__]) +# self._dispatch(v) +# self._write(")") +# +# def _Attribute(self,t): +# self._dispatch(t.value) +# self._write(".") +# self._write(t.attr) +# +## def _Call(self, t): +## self._dispatch(t.func) +## self._write("(") +## comma = False +## for e in t.args: +## if comma: self._write(", ") +## else: comma = True +## self._dispatch(e) +## for e in t.keywords: +## if comma: self._write(", ") +## else: comma = True +## self._dispatch(e) +## if t.starargs: +## if comma: self._write(", ") +## else: comma = True +## self._write("*") +## self._dispatch(t.starargs) +## if t.kwargs: +## if comma: self._write(", ") +## else: comma = True +## self._write("**") +## self._dispatch(t.kwargs) +## self._write(")") +# +# # slice +# def _Index(self, t): +# self._dispatch(t.value) +# +# def _ExtSlice(self, t): +# for i, d in enumerate(t.dims): +# if i != 0: +# self._write(': ') +# self._dispatch(d) +# +# # others +# def _arguments(self, t): +# first = True +# nonDef = len(t.args)-len(t.defaults) +# for a in t.args[0:nonDef]: +# if first:first = False +# else: self._write(", ") +# self._dispatch(a) +# for a,d in zip(t.args[nonDef:], t.defaults): +# if first:first = False +# else: self._write(", ") +# self._dispatch(a), +# self._write("=") +# self._dispatch(d) +# if t.vararg: +# if first:first = False +# else: self._write(", ") +# self._write("*"+t.vararg) +# if t.kwarg: +# if first:first = False +# else: self._write(", ") +# self._write("**"+t.kwarg) +# +## def _keyword(self, t): +## self._write(t.arg) +## self._write("=") +## self._dispatch(t.value) +# +# def _Lambda(self, t): +# self._write("lambda ") +# self._dispatch(t.args) +# self._write(": ") +# self._dispatch(t.body) + + + diff --git a/astropy_helpers/sphinx/ext/docscrape.py b/astropy_helpers/sphinx/ext/docscrape.py new file mode 100644 index 00000000..3d92b983 --- /dev/null +++ b/astropy_helpers/sphinx/ext/docscrape.py @@ -0,0 +1,508 @@ +"""Extract reference documentation from the NumPy source tree. + +""" + +import inspect +import textwrap +import re +import pydoc +from StringIO import StringIO + +class Reader(object): + """A line-based string reader. + + """ + def __init__(self, data): + """ + Parameters + ---------- + data : str + String with lines separated by '\n'. + + """ + if isinstance(data,list): + self._str = data + else: + self._str = data.split('\n') # store string as list of lines + + self.reset() + + def __getitem__(self, n): + return self._str[n] + + def reset(self): + self._l = 0 # current line nr + + def read(self): + if not self.eof(): + out = self[self._l] + self._l += 1 + return out + else: + return '' + + def seek_next_non_empty_line(self): + for l in self[self._l:]: + if l.strip(): + break + else: + self._l += 1 + + def eof(self): + return self._l >= len(self._str) + + def read_to_condition(self, condition_func): + start = self._l + for line in self[start:]: + if condition_func(line): + return self[start:self._l] + self._l += 1 + if self.eof(): + return self[start:self._l+1] + return [] + + def read_to_next_empty_line(self): + self.seek_next_non_empty_line() + def is_empty(line): + return not line.strip() + return self.read_to_condition(is_empty) + + def read_to_next_unindented_line(self): + def is_unindented(line): + return (line.strip() and (len(line.lstrip()) == len(line))) + return self.read_to_condition(is_unindented) + + def peek(self,n=0): + if self._l + n < len(self._str): + return self[self._l + n] + else: + return '' + + def is_empty(self): + return not ''.join(self._str).strip() + + +class NumpyDocString(object): + def __init__(self, docstring, config={}, warn=None): + from warnings import warn as stdlib_warn + + self.warn = stdlib_warn if warn is None else warn + + docstring = textwrap.dedent(docstring).split('\n') + self._doc = Reader(docstring) + self._parsed_data = { + 'Signature': '', + 'Summary': [''], + 'Extended Summary': [], + 'Parameters': [], + 'Returns': [], + 'Raises': [], + 'Warns': [], + 'Other Parameters': [], + 'Attributes': [], + 'Methods': [], + 'See Also': [], + 'Notes': [], + 'Warnings': [], + 'References': '', + 'Examples': '', + 'index': {} + } + + self._parse() + + + def __getitem__(self,key): + return self._parsed_data[key] + + def __setitem__(self,key,val): + if not self._parsed_data.has_key(key): + self.warn("Unknown section %s" % key) + else: + self._parsed_data[key] = val + + def _is_at_section(self): + self._doc.seek_next_non_empty_line() + + if self._doc.eof(): + return False + + l1 = self._doc.peek().strip() # e.g. Parameters + + if l1.startswith('.. index::'): + return True + + l2 = self._doc.peek(1).strip() # ---------- or ========== + return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1)) + + def _strip(self,doc): + i = 0 + j = 0 + for i,line in enumerate(doc): + if line.strip(): break + + for j,line in enumerate(doc[::-1]): + if line.strip(): break + + return doc[i:len(doc)-j] + + def _read_to_next_section(self): + section = self._doc.read_to_next_empty_line() + + while not self._is_at_section() and not self._doc.eof(): + if not self._doc.peek(-1).strip(): # previous line was empty + section += [''] + + section += self._doc.read_to_next_empty_line() + + return section + + def _read_sections(self): + while not self._doc.eof(): + data = self._read_to_next_section() + name = data[0].strip() + + if name.startswith('..'): # index section + yield name, data[1:] + elif len(data) < 2: + yield StopIteration + else: + yield name, self._strip(data[2:]) + + def _parse_param_list(self,content): + r = Reader(content) + params = [] + while not r.eof(): + header = r.read().strip() + if ' : ' in header: + arg_name, arg_type = header.split(' : ')[:2] + else: + arg_name, arg_type = header, '' + + desc = r.read_to_next_unindented_line() + desc = dedent_lines(desc) + + params.append((arg_name,arg_type,desc)) + + return params + + + _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" + r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) + def _parse_see_also(self, content): + """ + func_name : Descriptive text + continued text + another_func_name : Descriptive text + func_name1, func_name2, :meth:`func_name`, func_name3 + + """ + items = [] + + def parse_item_name(text): + """Match ':role:`name`' or 'name'""" + m = self._name_rgx.match(text) + if m: + g = m.groups() + if g[1] is None: + return g[3], None + else: + return g[2], g[1] + raise ValueError("%s is not a item name" % text) + + def push_item(name, rest): + if not name: + return + name, role = parse_item_name(name) + items.append((name, list(rest), role)) + del rest[:] + + current_func = None + rest = [] + + for line in content: + if not line.strip(): continue + + m = self._name_rgx.match(line) + if m and line[m.end():].strip().startswith(':'): + push_item(current_func, rest) + current_func, line = line[:m.end()], line[m.end():] + rest = [line.split(':', 1)[1].strip()] + if not rest[0]: + rest = [] + elif not line.startswith(' '): + push_item(current_func, rest) + current_func = None + if ',' in line: + for func in line.split(','): + if func.strip(): + push_item(func, []) + elif line.strip(): + current_func = line + elif current_func is not None: + rest.append(line.strip()) + push_item(current_func, rest) + return items + + def _parse_index(self, section, content): + """ + .. index: default + :refguide: something, else, and more + + """ + def strip_each_in(lst): + return [s.strip() for s in lst] + + out = {} + section = section.split('::') + if len(section) > 1: + out['default'] = strip_each_in(section[1].split(','))[0] + for line in content: + line = line.split(':') + if len(line) > 2: + out[line[1]] = strip_each_in(line[2].split(',')) + return out + + def _parse_summary(self): + """Grab signature (if given) and summary""" + if self._is_at_section(): + return + + summary = self._doc.read_to_next_empty_line() + summary_str = " ".join([s.strip() for s in summary]).strip() + if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str): + self['Signature'] = summary_str + if not self._is_at_section(): + self['Summary'] = self._doc.read_to_next_empty_line() + else: + self['Summary'] = summary + + if not self._is_at_section(): + self['Extended Summary'] = self._read_to_next_section() + + def _parse(self): + self._doc.reset() + self._parse_summary() + + for (section,content) in self._read_sections(): + if not section.startswith('..'): + section = ' '.join([s.capitalize() for s in section.split(' ')]) + if section in ('Parameters', 'Returns', 'Raises', 'Warns', + 'Other Parameters', 'Attributes', 'Methods'): + self[section] = self._parse_param_list(content) + elif section.startswith('.. index::'): + self['index'] = self._parse_index(section, content) + elif section == 'See Also': + self['See Also'] = self._parse_see_also(content) + else: + self[section] = content + + # string conversion routines + + def _str_header(self, name, symbol='-'): + return [name, len(name)*symbol] + + def _str_indent(self, doc, indent=4): + out = [] + for line in doc: + out += [' '*indent + line] + return out + + def _str_signature(self): + if self['Signature']: + return [self['Signature'].replace('*','\*')] + [''] + else: + return [''] + + def _str_summary(self): + if self['Summary']: + return self['Summary'] + [''] + else: + return [] + + def _str_extended_summary(self): + if self['Extended Summary']: + return self['Extended Summary'] + [''] + else: + return [] + + def _str_param_list(self, name): + out = [] + if self[name]: + out += self._str_header(name) + for param,param_type,desc in self[name]: + out += ['%s : %s' % (param, param_type)] + out += self._str_indent(desc) + out += [''] + return out + + def _str_section(self, name): + out = [] + if self[name]: + out += self._str_header(name) + out += self[name] + out += [''] + return out + + def _str_see_also(self, func_role): + if not self['See Also']: return [] + out = [] + out += self._str_header("See Also") + last_had_desc = True + for func, desc, role in self['See Also']: + if role: + link = ':%s:`%s`' % (role, func) + elif func_role: + link = ':%s:`%s`' % (func_role, func) + else: + link = "`%s`_" % func + if desc or last_had_desc: + out += [''] + out += [link] + else: + out[-1] += ", %s" % link + if desc: + out += self._str_indent([' '.join(desc)]) + last_had_desc = True + else: + last_had_desc = False + out += [''] + return out + + def _str_index(self): + idx = self['index'] + out = [] + out += ['.. index:: %s' % idx.get('default','')] + for section, references in idx.iteritems(): + if section == 'default': + continue + out += [' :%s: %s' % (section, ', '.join(references))] + return out + + def __str__(self, func_role=''): + out = [] + out += self._str_signature() + out += self._str_summary() + out += self._str_extended_summary() + for param_list in ('Parameters', 'Returns', 'Other Parameters', + 'Raises', 'Warns'): + out += self._str_param_list(param_list) + out += self._str_section('Warnings') + out += self._str_see_also(func_role) + for s in ('Notes','References','Examples'): + out += self._str_section(s) + for param_list in ('Attributes', 'Methods'): + out += self._str_param_list(param_list) + out += self._str_index() + return '\n'.join(out) + + +def indent(str,indent=4): + indent_str = ' '*indent + if str is None: + return indent_str + lines = str.split('\n') + return '\n'.join(indent_str + l for l in lines) + +def dedent_lines(lines): + """Deindent a list of lines maximally""" + return textwrap.dedent("\n".join(lines)).split("\n") + +def header(text, style='-'): + return text + '\n' + style*len(text) + '\n' + + +class FunctionDoc(NumpyDocString): + def __init__(self, func, role='func', doc=None, config={}): + self._f = func + self._role = role # e.g. "func" or "meth" + + if doc is None: + if func is None: + raise ValueError("No function or docstring given") + doc = inspect.getdoc(func) or '' + NumpyDocString.__init__(self, doc) + + if not self['Signature'] and func is not None: + func, func_name = self.get_func() + try: + # try to read signature + argspec = inspect.getargspec(func) + argspec = inspect.formatargspec(*argspec) + argspec = argspec.replace('*','\*') + signature = '%s%s' % (func_name, argspec) + except TypeError, e: + signature = '%s()' % func_name + self['Signature'] = signature + + def get_func(self): + func_name = getattr(self._f, '__name__', self.__class__.__name__) + if inspect.isclass(self._f): + func = getattr(self._f, '__call__', self._f.__init__) + else: + func = self._f + return func, func_name + + def __str__(self): + out = '' + + func, func_name = self.get_func() + signature = self['Signature'].replace('*', '\*') + + roles = {'func': 'function', + 'meth': 'method'} + + if self._role: + if not roles.has_key(self._role): + self.warn("Warning: invalid role %s" % self._role) + out += '.. %s:: %s\n \n\n' % (roles.get(self._role,''), + func_name) + + out += super(FunctionDoc, self).__str__(func_role=self._role) + return out + + +class ClassDoc(NumpyDocString): + + extra_public_methods = ['__call__'] + + def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, + config={}): + if not inspect.isclass(cls) and cls is not None: + raise ValueError("Expected a class or None, but got %r" % cls) + self._cls = cls + + if modulename and not modulename.endswith('.'): + modulename += '.' + self._mod = modulename + + if doc is None: + if cls is None: + raise ValueError("No class or documentation string given") + doc = pydoc.getdoc(cls) + + NumpyDocString.__init__(self, doc) + + if config.get('show_class_members', True): + if not self['Methods']: + self['Methods'] = [(name, '', '') + for name in sorted(self.methods)] + if not self['Attributes']: + self['Attributes'] = [(name, '', '') + for name in sorted(self.properties)] + + @property + def methods(self): + if self._cls is None: + return [] + return [name for name,func in inspect.getmembers(self._cls) + if ((not name.startswith('_') + or name in self.extra_public_methods) + and callable(func))] + + @property + def properties(self): + if self._cls is None: + return [] + return [name for name,func in inspect.getmembers(self._cls) + if not name.startswith('_') and func is None] diff --git a/astropy_helpers/sphinx/ext/docscrape_sphinx.py b/astropy_helpers/sphinx/ext/docscrape_sphinx.py new file mode 100644 index 00000000..4b4d094d --- /dev/null +++ b/astropy_helpers/sphinx/ext/docscrape_sphinx.py @@ -0,0 +1,227 @@ +import re, inspect, textwrap, pydoc +import sphinx +from docscrape import NumpyDocString, FunctionDoc, ClassDoc + +class SphinxDocString(NumpyDocString): + def __init__(self, docstring, config={}, warn=None): + self.use_plots = config.get('use_plots', False) + NumpyDocString.__init__(self, docstring, config=config, warn=warn) + + # string conversion routines + def _str_header(self, name, symbol='`'): + return ['.. rubric:: ' + name, ''] + + def _str_field_list(self, name): + return [':' + name + ':'] + + def _str_indent(self, doc, indent=4): + out = [] + for line in doc: + out += [' '*indent + line] + return out + + def _str_signature(self): + return [''] + if self['Signature']: + return ['``%s``' % self['Signature']] + [''] + else: + return [''] + + def _str_summary(self): + return self['Summary'] + [''] + + def _str_extended_summary(self): + return self['Extended Summary'] + [''] + + def _str_param_list(self, name): + out = [] + if self[name]: + out += self._str_field_list(name) + out += [''] + for param,param_type,desc in self[name]: + out += self._str_indent(['**%s** : %s' % (param.strip(), + param_type)]) + out += [''] + out += self._str_indent(desc,8) + out += [''] + return out + + @property + def _obj(self): + if hasattr(self, '_cls'): + return self._cls + elif hasattr(self, '_f'): + return self._f + return None + + def _str_member_list(self, name): + """ + Generate a member listing, autosummary:: table where possible, + and a table where not. + + """ + out = [] + if self[name]: + out += ['.. rubric:: %s' % name, ''] + prefix = getattr(self, '_name', '') + + if prefix: + prefix = '~%s.' % prefix + + autosum = [] + others = [] + for param, param_type, desc in self[name]: + param = param.strip() + if not self._obj or hasattr(self._obj, param): + autosum += [" %s%s" % (prefix, param)] + else: + others.append((param, param_type, desc)) + + if autosum: + out += ['.. autosummary::', ' :toctree:', ''] + out += autosum + + if others: + maxlen_0 = max([len(x[0]) for x in others]) + maxlen_1 = max([len(x[1]) for x in others]) + hdr = "="*maxlen_0 + " " + "="*maxlen_1 + " " + "="*10 + fmt = '%%%ds %%%ds ' % (maxlen_0, maxlen_1) + n_indent = maxlen_0 + maxlen_1 + 4 + out += [hdr] + for param, param_type, desc in others: + out += [fmt % (param.strip(), param_type)] + out += self._str_indent(desc, n_indent) + out += [hdr] + out += [''] + return out + + def _str_section(self, name): + out = [] + if self[name]: + out += self._str_header(name) + out += [''] + content = textwrap.dedent("\n".join(self[name])).split("\n") + out += content + out += [''] + return out + + def _str_see_also(self, func_role): + out = [] + if self['See Also']: + see_also = super(SphinxDocString, self)._str_see_also(func_role) + out = ['.. seealso::', ''] + out += self._str_indent(see_also[2:]) + return out + + def _str_warnings(self): + out = [] + if self['Warnings']: + out = ['.. warning::', ''] + out += self._str_indent(self['Warnings']) + return out + + def _str_index(self): + idx = self['index'] + out = [] + if len(idx) == 0: + return out + + out += ['.. index:: %s' % idx.get('default','')] + for section, references in idx.iteritems(): + if section == 'default': + continue + elif section == 'refguide': + out += [' single: %s' % (', '.join(references))] + else: + out += [' %s: %s' % (section, ','.join(references))] + return out + + def _str_references(self): + out = [] + if self['References']: + out += self._str_header('References') + if isinstance(self['References'], str): + self['References'] = [self['References']] + out.extend(self['References']) + out += [''] + # Latex collects all references to a separate bibliography, + # so we need to insert links to it + if sphinx.__version__ >= "0.6": + out += ['.. only:: latex',''] + else: + out += ['.. latexonly::',''] + items = [] + for line in self['References']: + m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I) + if m: + items.append(m.group(1)) + out += [' ' + ", ".join(["[%s]_" % item for item in items]), ''] + return out + + def _str_examples(self): + examples_str = "\n".join(self['Examples']) + + if (self.use_plots and 'import matplotlib' in examples_str + and 'plot::' not in examples_str): + out = [] + out += self._str_header('Examples') + out += ['.. plot::', ''] + out += self._str_indent(self['Examples']) + out += [''] + return out + else: + return self._str_section('Examples') + + def __str__(self, indent=0, func_role="obj"): + out = [] + out += self._str_signature() + out += self._str_index() + [''] + out += self._str_summary() + out += self._str_extended_summary() + for param_list in ('Parameters', 'Returns', 'Other Parameters', + 'Raises', 'Warns'): + out += self._str_param_list(param_list) + out += self._str_warnings() + out += self._str_see_also(func_role) + out += self._str_section('Notes') + out += self._str_references() + out += self._str_examples() + for param_list in ('Attributes', 'Methods'): + out += self._str_member_list(param_list) + out = self._str_indent(out,indent) + return '\n'.join(out) + +class SphinxFunctionDoc(SphinxDocString, FunctionDoc): + def __init__(self, obj, doc=None, config={}): + self.use_plots = config.get('use_plots', False) + FunctionDoc.__init__(self, obj, doc=doc, config=config) + +class SphinxClassDoc(SphinxDocString, ClassDoc): + def __init__(self, obj, doc=None, func_doc=None, config={}): + self.use_plots = config.get('use_plots', False) + ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config) + +class SphinxObjDoc(SphinxDocString): + def __init__(self, obj, doc=None, config={}): + self._f = obj + SphinxDocString.__init__(self, doc, config=config) + +def get_doc_object(obj, what=None, doc=None, config={}): + if what is None: + if inspect.isclass(obj): + what = 'class' + elif inspect.ismodule(obj): + what = 'module' + elif callable(obj): + what = 'function' + else: + what = 'object' + if what == 'class': + return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, + config=config) + elif what in ('function', 'method'): + return SphinxFunctionDoc(obj, doc=doc, config=config) + else: + if doc is None: + doc = pydoc.getdoc(obj) + return SphinxObjDoc(obj, doc, config=config) diff --git a/astropy_helpers/sphinx/ext/doctest.py b/astropy_helpers/sphinx/ext/doctest.py new file mode 100644 index 00000000..22f0cba2 --- /dev/null +++ b/astropy_helpers/sphinx/ext/doctest.py @@ -0,0 +1,33 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This is a set of three directives that allow us to insert metadata +about doctests into the .rst files so the testing framework knows +which tests to skip. + +This is quite different from the doctest extension in Sphinx itself, +which actually does something. For astropy, all of the testing is +centrally managed from py.test and Sphinx is not used for running +tests. +""" +from docutils.nodes import literal_block +from sphinx.util.compat import Directive + + +class DoctestSkipDirective(Directive): + has_content = True + + def run(self): + code = '\n'.join(self.content) + return [literal_block(code, code)] + + +class DoctestRequiresDirective(DoctestSkipDirective): + # This is silly, but we really support an unbounded number of + # optional arguments + optional_arguments = 64 + + +def setup(app): + app.add_directive('doctest-requires', DoctestRequiresDirective) + app.add_directive('doctest-skip', DoctestSkipDirective) + app.add_directive('doctest-skip-all', DoctestSkipDirective) diff --git a/astropy_helpers/sphinx/ext/edit_on_github.py b/astropy_helpers/sphinx/ext/edit_on_github.py new file mode 100644 index 00000000..dc02cdf4 --- /dev/null +++ b/astropy_helpers/sphinx/ext/edit_on_github.py @@ -0,0 +1,164 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This extension makes it easy to edit documentation on github. + +It adds links associated with each docstring that go to the +corresponding view source page on Github. From there, the user can +push the "Edit" button, edit the docstring, and submit a pull request. + +It has the following configuration options (to be set in the project's +``conf.py``): + +* `edit_on_github_project` + The name of the github project, in the form + "username/projectname". + +* `edit_on_github_branch` + The name of the branch to edit. If this is a released version, + this should be a git tag referring to that version. For a + dev version, it often makes sense for it to be "master". It + may also be a git hash. + +* `edit_on_github_source_root` + The location within the source tree of the root of the + Python package. Defaults to "lib". + +* `edit_on_github_doc_root` + The location within the source tree of the root of the + documentation source. Defaults to "doc", but it may make sense to + set it to "doc/source" if the project uses a separate source + directory. + +* `edit_on_github_docstring_message` + The phrase displayed in the links to edit a docstring. Defaults + to "[edit on github]". + +* `edit_on_github_page_message` + The phrase displayed in the links to edit a RST page. Defaults + to "[edit this page on github]". + +* `edit_on_github_help_message` + The phrase displayed as a tooltip on the edit links. Defaults to + "Push the Edit button on the next page" + +* `edit_on_github_skip_regex` + When the path to the .rst file matches this regular expression, + no "edit this page on github" link will be added. Defaults to + ``"_.*"``. +""" +import inspect +import os +import re +import sys + +from docutils import nodes + +from sphinx import addnodes + + +def import_object(modname, name): + """ + Import the object given by *modname* and *name* and return it. + If not found, or the import fails, returns None. + """ + try: + __import__(modname) + mod = sys.modules[modname] + obj = mod + for part in name.split('.'): + obj = getattr(obj, part) + return obj + except: + return None + + +def get_url_base(app): + return 'http://github.com/%s/tree/%s/' % ( + app.config.edit_on_github_project, + app.config.edit_on_github_branch) + + +def doctree_read(app, doctree): + # Get the configuration parameters + if app.config.edit_on_github_project == 'REQUIRED': + raise ValueError( + "The edit_on_github_project configuration variable must be " + "provided in the conf.py") + + source_root = app.config.edit_on_github_source_root + url = get_url_base(app) + + docstring_message = app.config.edit_on_github_docstring_message + + # Handle the docstring-editing links + for objnode in doctree.traverse(addnodes.desc): + if objnode.get('domain') != 'py': + continue + names = set() + for signode in objnode: + if not isinstance(signode, addnodes.desc_signature): + continue + modname = signode.get('module') + if not modname: + continue + fullname = signode.get('fullname') + if fullname in names: + # only one link per name, please + continue + names.add(fullname) + obj = import_object(modname, fullname) + anchor = None + if obj is not None: + try: + lines, lineno = inspect.getsourcelines(obj) + except: + pass + else: + anchor = '#L%d' % lineno + if anchor: + path = '%s%s%s.py%s' % ( + url, source_root, modname.replace('.', '/'), anchor) + onlynode = addnodes.only(expr='html') + onlynode += nodes.reference( + reftitle=app.config.edit_on_github_help_message, + refuri=path) + onlynode[0] += nodes.inline( + '', '', nodes.raw('', ' ', format='html'), + nodes.Text(docstring_message), + classes=['edit-on-github', 'viewcode-link']) + signode += onlynode + + +def html_page_context(app, pagename, templatename, context, doctree): + if (templatename == 'page.html' and + not re.match(app.config.edit_on_github_skip_regex, pagename)): + + doc_root = app.config.edit_on_github_doc_root + if doc_root != '' and not doc_root.endswith('/'): + doc_root += '/' + doc_path = os.path.relpath(doctree.get('source'), app.builder.srcdir) + url = get_url_base(app) + + page_message = app.config.edit_on_github_page_message + + context['edit_on_github'] = url + doc_root + doc_path + context['edit_on_github_page_message'] = ( + app.config.edit_on_github_page_message) + + +def setup(app): + app.add_config_value('edit_on_github_project', 'REQUIRED', True) + app.add_config_value('edit_on_github_branch', 'master', True) + app.add_config_value('edit_on_github_source_root', 'lib', True) + app.add_config_value('edit_on_github_doc_root', 'doc', True) + app.add_config_value('edit_on_github_docstring_message', + '[edit on github]', True) + app.add_config_value('edit_on_github_page_message', + 'Edit This Page on Github', True) + app.add_config_value('edit_on_github_help_message', + 'Push the Edit button on the next page', True) + app.add_config_value('edit_on_github_skip_regex', + '_.*', True) + + app.connect('doctree-read', doctree_read) + app.connect('html-page-context', html_page_context) diff --git a/astropy_helpers/sphinx/ext/numpydoc.py b/astropy_helpers/sphinx/ext/numpydoc.py new file mode 100644 index 00000000..7e6aa333 --- /dev/null +++ b/astropy_helpers/sphinx/ext/numpydoc.py @@ -0,0 +1,169 @@ +""" +======== +numpydoc +======== + +Sphinx extension that handles docstrings in the Numpy standard format. [1] + +It will: + +- Convert Parameters etc. sections to field lists. +- Convert See Also section to a See also entry. +- Renumber references. +- Extract the signature from the docstring, if it can't be determined otherwise. + +.. [1] http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard + +""" + +import sphinx + +if sphinx.__version__ < '1.0.1': + raise RuntimeError("Sphinx 1.0.1 or newer is required") + +import os, re, pydoc +from docscrape_sphinx import get_doc_object, SphinxDocString +from sphinx.util.compat import Directive +import inspect + +def mangle_docstrings(app, what, name, obj, options, lines, + reference_offset=[0]): + + cfg = dict(use_plots=app.config.numpydoc_use_plots, + show_class_members=app.config.numpydoc_show_class_members) + + if what == 'module': + # Strip top title + title_re = re.compile(ur'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*', + re.I|re.S) + lines[:] = title_re.sub(u'', u"\n".join(lines)).split(u"\n") + else: + doc = get_doc_object(obj, what, u"\n".join(lines), config=cfg) + lines[:] = unicode(doc).split(u"\n") + + if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \ + obj.__name__: + if hasattr(obj, '__module__'): + v = dict(full_name=u"%s.%s" % (obj.__module__, obj.__name__)) + else: + v = dict(full_name=obj.__name__) + lines += [u'', u'.. htmlonly::', ''] + lines += [u' %s' % x for x in + (app.config.numpydoc_edit_link % v).split("\n")] + + # replace reference numbers so that there are no duplicates + references = [] + for line in lines: + line = line.strip() + m = re.match(ur'^.. \[([a-z0-9_.-])\]', line, re.I) + if m: + references.append(m.group(1)) + + # start renaming from the longest string, to avoid overwriting parts + references.sort(key=lambda x: -len(x)) + if references: + for i, line in enumerate(lines): + for r in references: + if re.match(ur'^\d+$', r): + new_r = u"R%d" % (reference_offset[0] + int(r)) + else: + new_r = u"%s%d" % (r, reference_offset[0]) + lines[i] = lines[i].replace(u'[%s]_' % r, + u'[%s]_' % new_r) + lines[i] = lines[i].replace(u'.. [%s]' % r, + u'.. [%s]' % new_r) + + reference_offset[0] += len(references) + +def mangle_signature(app, what, name, obj, options, sig, retann): + # Do not try to inspect classes that don't define `__init__` + if (inspect.isclass(obj) and + (not hasattr(obj, '__init__') or + 'initializes x; see ' in pydoc.getdoc(obj.__init__))): + return '', '' + + if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')): return + if not hasattr(obj, '__doc__'): return + + doc = SphinxDocString(pydoc.getdoc(obj), warn=app.warn) + if doc['Signature']: + sig = re.sub(u"^[^(]*", u"", doc['Signature']) + return sig, u'' + +def setup(app, get_doc_object_=get_doc_object): + global get_doc_object + get_doc_object = get_doc_object_ + + app.connect('autodoc-process-docstring', mangle_docstrings) + app.connect('autodoc-process-signature', mangle_signature) + app.add_config_value('numpydoc_edit_link', None, False) + app.add_config_value('numpydoc_use_plots', None, False) + app.add_config_value('numpydoc_show_class_members', True, True) + + # Extra mangling domains + app.add_domain(NumpyPythonDomain) + app.add_domain(NumpyCDomain) + +#------------------------------------------------------------------------------ +# Docstring-mangling domains +#------------------------------------------------------------------------------ + +from docutils.statemachine import ViewList +from sphinx.domains.c import CDomain +from sphinx.domains.python import PythonDomain + +class ManglingDomainBase(object): + directive_mangling_map = {} + + def __init__(self, *a, **kw): + super(ManglingDomainBase, self).__init__(*a, **kw) + self.wrap_mangling_directives() + + def wrap_mangling_directives(self): + for name, objtype in self.directive_mangling_map.items(): + self.directives[name] = wrap_mangling_directive( + self.directives[name], objtype) + +class NumpyPythonDomain(ManglingDomainBase, PythonDomain): + name = 'np' + directive_mangling_map = { + 'function': 'function', + 'class': 'class', + 'exception': 'class', + 'method': 'function', + 'classmethod': 'function', + 'staticmethod': 'function', + 'attribute': 'attribute', + } + +class NumpyCDomain(ManglingDomainBase, CDomain): + name = 'np-c' + directive_mangling_map = { + 'function': 'function', + 'member': 'attribute', + 'macro': 'function', + 'type': 'class', + 'var': 'object', + } + +def wrap_mangling_directive(base_directive, objtype): + class directive(base_directive): + def run(self): + env = self.state.document.settings.env + + name = None + if self.arguments: + m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0]) + name = m.group(2).strip() + + if not name: + name = self.arguments[0] + + lines = list(self.content) + mangle_docstrings(env.app, objtype, name, None, None, lines) + self.content = ViewList(lines, self.content.parent) + + return base_directive.run(self) + + return directive + diff --git a/astropy_helpers/sphinx/ext/phantom_import.py b/astropy_helpers/sphinx/ext/phantom_import.py new file mode 100644 index 00000000..c77eeb54 --- /dev/null +++ b/astropy_helpers/sphinx/ext/phantom_import.py @@ -0,0 +1,162 @@ +""" +============== +phantom_import +============== + +Sphinx extension to make directives from ``sphinx.ext.autodoc`` and similar +extensions to use docstrings loaded from an XML file. + +This extension loads an XML file in the Pydocweb format [1] and +creates a dummy module that contains the specified docstrings. This +can be used to get the current docstrings from a Pydocweb instance +without needing to rebuild the documented module. + +.. [1] http://code.google.com/p/pydocweb + +""" +import imp, sys, compiler, types, os, inspect, re + +def setup(app): + app.connect('builder-inited', initialize) + app.add_config_value('phantom_import_file', None, True) + +def initialize(app): + fn = app.config.phantom_import_file + if (fn and os.path.isfile(fn)): + print "[numpydoc] Phantom importing modules from", fn, "..." + import_phantom_module(fn) + +#------------------------------------------------------------------------------ +# Creating 'phantom' modules from an XML description +#------------------------------------------------------------------------------ +def import_phantom_module(xml_file): + """ + Insert a fake Python module to sys.modules, based on a XML file. + + The XML file is expected to conform to Pydocweb DTD. The fake + module will contain dummy objects, which guarantee the following: + + - Docstrings are correct. + - Class inheritance relationships are correct (if present in XML). + - Function argspec is *NOT* correct (even if present in XML). + Instead, the function signature is prepended to the function docstring. + - Class attributes are *NOT* correct; instead, they are dummy objects. + + Parameters + ---------- + xml_file : str + Name of an XML file to read + + """ + import lxml.etree as etree + + object_cache = {} + + tree = etree.parse(xml_file) + root = tree.getroot() + + # Sort items so that + # - Base classes come before classes inherited from them + # - Modules come before their contents + all_nodes = dict([(n.attrib['id'], n) for n in root]) + + def _get_bases(node, recurse=False): + bases = [x.attrib['ref'] for x in node.findall('base')] + if recurse: + j = 0 + while True: + try: + b = bases[j] + except IndexError: break + if b in all_nodes: + bases.extend(_get_bases(all_nodes[b])) + j += 1 + return bases + + type_index = ['module', 'class', 'callable', 'object'] + + def base_cmp(a, b): + x = cmp(type_index.index(a.tag), type_index.index(b.tag)) + if x != 0: return x + + if a.tag == 'class' and b.tag == 'class': + a_bases = _get_bases(a, recurse=True) + b_bases = _get_bases(b, recurse=True) + x = cmp(len(a_bases), len(b_bases)) + if x != 0: return x + if a.attrib['id'] in b_bases: return -1 + if b.attrib['id'] in a_bases: return 1 + + return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.')) + + nodes = root.getchildren() + nodes.sort(base_cmp) + + # Create phantom items + for node in nodes: + name = node.attrib['id'] + doc = (node.text or '').decode('string-escape') + "\n" + if doc == "\n": doc = "" + + # create parent, if missing + parent = name + while True: + parent = '.'.join(parent.split('.')[:-1]) + if not parent: break + if parent in object_cache: break + obj = imp.new_module(parent) + object_cache[parent] = obj + sys.modules[parent] = obj + + # create object + if node.tag == 'module': + obj = imp.new_module(name) + obj.__doc__ = doc + sys.modules[name] = obj + elif node.tag == 'class': + bases = [object_cache[b] for b in _get_bases(node) + if b in object_cache] + bases.append(object) + init = lambda self: None + init.__doc__ = doc + obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init}) + obj.__name__ = name.split('.')[-1] + elif node.tag == 'callable': + funcname = node.attrib['id'].split('.')[-1] + argspec = node.attrib.get('argspec') + if argspec: + argspec = re.sub('^[^(]*', '', argspec) + doc = "%s%s\n\n%s" % (funcname, argspec, doc) + obj = lambda: 0 + obj.__argspec_is_invalid_ = True + obj.func_name = funcname + obj.__name__ = name + obj.__doc__ = doc + if inspect.isclass(object_cache[parent]): + obj.__objclass__ = object_cache[parent] + else: + class Dummy(object): pass + obj = Dummy() + obj.__name__ = name + obj.__doc__ = doc + if inspect.isclass(object_cache[parent]): + obj.__get__ = lambda: None + object_cache[name] = obj + + if parent: + if inspect.ismodule(object_cache[parent]): + obj.__module__ = parent + setattr(object_cache[parent], name.split('.')[-1], obj) + + # Populate items + for node in root: + obj = object_cache.get(node.attrib['id']) + if obj is None: continue + for ref in node.findall('ref'): + if node.tag == 'class': + if ref.attrib['ref'].startswith(node.attrib['id'] + '.'): + setattr(obj, ref.attrib['name'], + object_cache.get(ref.attrib['ref'])) + else: + setattr(obj, ref.attrib['name'], + object_cache.get(ref.attrib['ref'])) diff --git a/astropy_helpers/sphinx/ext/templates/autosummary_core/base.rst b/astropy_helpers/sphinx/ext/templates/autosummary_core/base.rst new file mode 100644 index 00000000..a58aa35f --- /dev/null +++ b/astropy_helpers/sphinx/ext/templates/autosummary_core/base.rst @@ -0,0 +1,10 @@ +{% if referencefile %} +.. include:: {{ referencefile }} +{% endif %} + +{{ objname }} +{{ underline }} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} diff --git a/astropy_helpers/sphinx/ext/templates/autosummary_core/class.rst b/astropy_helpers/sphinx/ext/templates/autosummary_core/class.rst new file mode 100644 index 00000000..85105fa8 --- /dev/null +++ b/astropy_helpers/sphinx/ext/templates/autosummary_core/class.rst @@ -0,0 +1,65 @@ +{% if referencefile %} +.. include:: {{ referencefile }} +{% endif %} + +{{ objname }} +{{ underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :show-inheritance: + + {% if '__init__' in methods %} + {% set caught_result = methods.remove('__init__') %} + {% endif %} + + {% block attributes_summary %} + {% if attributes %} + + .. rubric:: Attributes Summary + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block methods_summary %} + {% if methods %} + + .. rubric:: Methods Summary + + .. autosummary:: + {% for item in methods %} + ~{{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block attributes_documentation %} + {% if attributes %} + + .. rubric:: Attributes Documentation + + {% for item in attributes %} + .. autoattribute:: {{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block methods_documentation %} + {% if methods %} + + .. rubric:: Methods Documentation + + {% for item in methods %} + .. automethod:: {{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} diff --git a/astropy_helpers/sphinx/ext/templates/autosummary_core/module.rst b/astropy_helpers/sphinx/ext/templates/autosummary_core/module.rst new file mode 100644 index 00000000..11208a25 --- /dev/null +++ b/astropy_helpers/sphinx/ext/templates/autosummary_core/module.rst @@ -0,0 +1,41 @@ +{% if referencefile %} +.. include:: {{ referencefile }} +{% endif %} + +{{ objname }} +{{ underline }} + +.. automodule:: {{ fullname }} + + {% block functions %} + {% if functions %} + .. rubric:: Functions + + .. autosummary:: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: Classes + + .. autosummary:: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: Exceptions + + .. autosummary:: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/astropy_helpers/sphinx/ext/tests/__init__.py b/astropy_helpers/sphinx/ext/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/astropy_helpers/sphinx/ext/tests/test_automodapi.py b/astropy_helpers/sphinx/ext/tests/test_automodapi.py new file mode 100644 index 00000000..84d42743 --- /dev/null +++ b/astropy_helpers/sphinx/ext/tests/test_automodapi.py @@ -0,0 +1,300 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import os + +import pytest + +pytest.importorskip('sphinx') # skips these tests if sphinx not present + + +class FakeConfig(object): + """ + Mocks up a sphinx configuration setting construct for automodapi tests + """ + def __init__(self, **kwargs): + for k, v in kwargs.iteritems(): + setattr(self, k, v) + + +class FakeApp(object): + """ + Mocks up a `sphinx.application.Application` object for automodapi tests + """ + + # Some default config values + _defaults = { + 'automodapi_toctreedirnm': 'api', + 'automodapi_writereprocessed': False + } + + def __init__(self, **configs): + config = self._defaults.copy() + config.update(configs) + self.config = FakeConfig(**config) + self.info = [] + self.warnings = [] + + def info(self, msg, loc): + self.info.append((msg, loc)) + + def warn(self, msg, loc): + self.warnings.append((msg, loc)) + + +am_replacer_str = """ +This comes before + +.. automodapi:: astropy_helpers.sphinx.ext.tests.test_automodapi +{options} + +This comes after +""" + +am_replacer_basic_expected = """ +This comes before + +astropy_helpers.sphinx.ext.tests.test_automodapi Module +------------------------------------------------------- + +.. automodule:: astropy_helpers.sphinx.ext.tests.test_automodapi + +Functions +^^^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :functions-only: + :toctree: api/ + {empty} + +Classes +^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :classes-only: + :toctree: api/ + {empty} + +Class Inheritance Diagram +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automod-diagram:: astropy_helpers.sphinx.ext.tests.test_automodapi + :private-bases: + +This comes after +""".format(empty='').replace('/', os.sep) +# the .format is necessary for editors that remove empty-line whitespace + + +def test_am_replacer_basic(): + """ + Tests replacing an ".. automodapi::" with the automodapi no-option + template + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + result = automodapi_replace(am_replacer_str.format(options=''), fakeapp) + + assert result == am_replacer_basic_expected + +am_replacer_noinh_expected = """ +This comes before + +astropy_helpers.sphinx.ext.tests.test_automodapi Module +------------------------------------------------------- + +.. automodule:: astropy_helpers.sphinx.ext.tests.test_automodapi + +Functions +^^^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :functions-only: + :toctree: api/ + {empty} + +Classes +^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :classes-only: + :toctree: api/ + {empty} + + +This comes after +""".format(empty='').replace('/', os.sep) + + +def test_am_replacer_noinh(): + """ + Tests replacing an ".. automodapi::" with no-inheritance-diagram + option + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + ops = ['', ':no-inheritance-diagram:'] + ostr = '\n '.join(ops) + result = automodapi_replace(am_replacer_str.format(options=ostr), fakeapp) + + assert result == am_replacer_noinh_expected + +am_replacer_titleandhdrs_expected = """ +This comes before + +astropy_helpers.sphinx.ext.tests.test_automodapi Module +&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& + +.. automodule:: astropy_helpers.sphinx.ext.tests.test_automodapi + +Functions +********* + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :functions-only: + :toctree: api/ + {empty} + +Classes +******* + +.. automodsumm:: astropy_helpers.sphinx.ext.tests.test_automodapi + :classes-only: + :toctree: api/ + {empty} + +Class Inheritance Diagram +************************* + +.. automod-diagram:: astropy_helpers.sphinx.ext.tests.test_automodapi + :private-bases: + + +This comes after +""".format(empty='').replace('/', os.sep) + + +def test_am_replacer_titleandhdrs(): + """ + Tests replacing an ".. automodapi::" entry with title-setting and header + character options. + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + ops = ['', ':title: A new title', ':headings: &*'] + ostr = '\n '.join(ops) + result = automodapi_replace(am_replacer_str.format(options=ostr), fakeapp) + + assert result == am_replacer_titleandhdrs_expected + + +am_replacer_nomain_str = """ +This comes before + +.. automodapi:: astropy_helpers.sphinx.ext.automodapi + :no-main-docstr: + +This comes after +""" + +am_replacer_nomain_expected = """ +This comes before + +astropy_helpers.sphinx.ext.automodapi Module +-------------------------------------------- + + + +Functions +^^^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.automodapi + :functions-only: + :toctree: api/ + {empty} + + +This comes after +""".format(empty='').replace('/', os.sep) + + +def test_am_replacer_nomain(): + """ + Tests replacing an ".. automodapi::" with "no-main-docstring" . + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + result = automodapi_replace(am_replacer_nomain_str, fakeapp) + + assert result == am_replacer_nomain_expected + + +am_replacer_skip_str = """ +This comes before + +.. automodapi:: astropy_helpers.sphinx.ext.automodapi + :skip: something1 + :skip: something2 + +This comes after +""" + +am_replacer_skip_expected = """ +This comes before + +astropy_helpers.sphinx.ext.automodapi Module +-------------------------------------------- + +.. automodule:: astropy_helpers.sphinx.ext.automodapi + +Functions +^^^^^^^^^ + +.. automodsumm:: astropy_helpers.sphinx.ext.automodapi + :functions-only: + :toctree: api/ + :skip: something1,something2 + + +This comes after +""".format(empty='').replace('/', os.sep) + + +def test_am_replacer_skip(): + """ + Tests using the ":skip: option in an ".. automodapi::" . + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + result = automodapi_replace(am_replacer_skip_str, fakeapp) + + assert result == am_replacer_skip_expected + + +am_replacer_invalidop_str = """ +This comes before + +.. automodapi:: astropy_helpers.sphinx.ext.automodapi + :invalid-option: + +This comes after +""" + + +def test_am_replacer_invalidop(): + """ + Tests that a sphinx warning is produced with an invalid option. + """ + from ..automodapi import automodapi_replace + + fakeapp = FakeApp() + automodapi_replace(am_replacer_invalidop_str, fakeapp) + + expected_warnings = [('Found additional options invalid-option in ' + 'automodapi.', None)] + + assert fakeapp.warnings == expected_warnings diff --git a/astropy_helpers/sphinx/ext/tests/test_automodsumm.py b/astropy_helpers/sphinx/ext/tests/test_automodsumm.py new file mode 100644 index 00000000..24abbfc7 --- /dev/null +++ b/astropy_helpers/sphinx/ext/tests/test_automodsumm.py @@ -0,0 +1,75 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest +pytest.importorskip('sphinx') # skips these tests if sphinx not present + + +class FakeEnv(object): + """ + Mocks up a sphinx env setting construct for automodapi tests + """ + def __init__(self, **kwargs): + for k, v in kwargs.iteritems(): + setattr(self, k, v) + + +class FakeBuilder(object): + """ + Mocks up a sphinx builder setting construct for automodapi tests + """ + def __init__(self, **kwargs): + self.env = FakeEnv(**kwargs) + + +class FakeApp(object): + """ + Mocks up a `sphinx.application.Application` object for automodapi tests + """ + def __init__(self, srcdir, automodapipresent=True): + self.builder = FakeBuilder(srcdir=srcdir) + self.info = [] + self.warnings = [] + self._extensions = [] + if automodapipresent: + self._extensions.append('astropy_helpers.sphinx.ext.automodapi') + + def info(self, msg, loc): + self.info.append((msg, loc)) + + def warn(self, msg, loc): + self.warnings.append((msg, loc)) + + +ams_to_asmry_str = """ +Before + +.. automodsumm:: astropy_helpers.sphinx.ext.automodsumm + :p: + +And After +""" + +ams_to_asmry_expected = """\ +.. currentmodule:: astropy_helpers.sphinx.ext.automodsumm + +.. autosummary:: + :p: + + Automoddiagram + Automodsumm + automodsumm_to_autosummary_lines + generate_automodsumm_docs + process_automodsumm_generation + setup""" + + +def test_ams_to_asmry(tmpdir): + from ..automodsumm import automodsumm_to_autosummary_lines + + fi = tmpdir.join('automodsumm.rst') + fi.write(ams_to_asmry_str) + + fakeapp = FakeApp(srcdir='') + resultlines = automodsumm_to_autosummary_lines(str(fi), fakeapp) + + assert '\n'.join(resultlines) == ams_to_asmry_expected diff --git a/astropy_helpers/sphinx/ext/tests/test_utils.py b/astropy_helpers/sphinx/ext/tests/test_utils.py new file mode 100644 index 00000000..44a72400 --- /dev/null +++ b/astropy_helpers/sphinx/ext/tests/test_utils.py @@ -0,0 +1,29 @@ +#namedtuple is needed for find_mod_objs so it can have a non-local module +from collections import namedtuple + +from ..utils import find_mod_objs + + +def test_find_mod_objs(): + lnms, fqns, objs = find_mod_objs('astropy') + + # this import is after the above call intentionally to make sure + # find_mod_objs properly imports astropy on its own + import astropy + + # just check for astropy.test ... other things might be added, so we + # shouldn't check that it's the only thing + assert 'test' in lnms + assert astropy.test in objs + + lnms, fqns, objs = find_mod_objs('astropy.utils.tests.test_misc', + onlylocals=False) + assert 'namedtuple' in lnms + assert 'collections.namedtuple' in fqns + assert namedtuple in objs + + lnms, fqns, objs = find_mod_objs('astropy.utils.tests.test_misc', + onlylocals=True) + assert 'namedtuple' not in lnms + assert 'collections.namedtuple' not in fqns + assert namedtuple not in objs diff --git a/astropy_helpers/sphinx/ext/tocdepthfix.py b/astropy_helpers/sphinx/ext/tocdepthfix.py new file mode 100644 index 00000000..be294788 --- /dev/null +++ b/astropy_helpers/sphinx/ext/tocdepthfix.py @@ -0,0 +1,18 @@ +from sphinx import addnodes + + +def fix_toc_entries(app, doctree): + # Get the docname; I don't know why this isn't just passed in to the + # callback + # This seems a bit unreliable as it's undocumented, but it's not "private" + # either: + docname = app.builder.env.temp_data['docname'] + if app.builder.env.metadata[docname].get('tocdepth', 0) != 0: + # We need to reprocess any TOC nodes in the doctree and make sure all + # the files listed in any TOCs are noted + for treenode in doctree.traverse(addnodes.toctree): + app.builder.env.note_toctree(docname, treenode) + + +def setup(app): + app.connect('doctree-read', fix_toc_entries) diff --git a/astropy_helpers/sphinx/ext/traitsdoc.py b/astropy_helpers/sphinx/ext/traitsdoc.py new file mode 100644 index 00000000..0fcf2c1c --- /dev/null +++ b/astropy_helpers/sphinx/ext/traitsdoc.py @@ -0,0 +1,140 @@ +""" +========= +traitsdoc +========= + +Sphinx extension that handles docstrings in the Numpy standard format, [1] +and support Traits [2]. + +This extension can be used as a replacement for ``numpydoc`` when support +for Traits is required. + +.. [1] http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard +.. [2] http://code.enthought.com/projects/traits/ + +""" + +import inspect +import os +import pydoc + +import docscrape +import docscrape_sphinx +from docscrape_sphinx import SphinxClassDoc, SphinxFunctionDoc, SphinxDocString + +import numpydoc + +import comment_eater + +class SphinxTraitsDoc(SphinxClassDoc): + def __init__(self, cls, modulename='', func_doc=SphinxFunctionDoc): + if not inspect.isclass(cls): + raise ValueError("Initialise using a class. Got %r" % cls) + self._cls = cls + + if modulename and not modulename.endswith('.'): + modulename += '.' + self._mod = modulename + self._name = cls.__name__ + self._func_doc = func_doc + + docstring = pydoc.getdoc(cls) + docstring = docstring.split('\n') + + # De-indent paragraph + try: + indent = min(len(s) - len(s.lstrip()) for s in docstring + if s.strip()) + except ValueError: + indent = 0 + + for n,line in enumerate(docstring): + docstring[n] = docstring[n][indent:] + + self._doc = docscrape.Reader(docstring) + self._parsed_data = { + 'Signature': '', + 'Summary': '', + 'Description': [], + 'Extended Summary': [], + 'Parameters': [], + 'Returns': [], + 'Raises': [], + 'Warns': [], + 'Other Parameters': [], + 'Traits': [], + 'Methods': [], + 'See Also': [], + 'Notes': [], + 'References': '', + 'Example': '', + 'Examples': '', + 'index': {} + } + + self._parse() + + def _str_summary(self): + return self['Summary'] + [''] + + def _str_extended_summary(self): + return self['Description'] + self['Extended Summary'] + [''] + + def __str__(self, indent=0, func_role="func"): + out = [] + out += self._str_signature() + out += self._str_index() + [''] + out += self._str_summary() + out += self._str_extended_summary() + for param_list in ('Parameters', 'Traits', 'Methods', + 'Returns','Raises'): + out += self._str_param_list(param_list) + out += self._str_see_also("obj") + out += self._str_section('Notes') + out += self._str_references() + out += self._str_section('Example') + out += self._str_section('Examples') + out = self._str_indent(out,indent) + return '\n'.join(out) + +def looks_like_issubclass(obj, classname): + """ Return True if the object has a class or superclass with the given class + name. + + Ignores old-style classes. + """ + t = obj + if t.__name__ == classname: + return True + for klass in t.__mro__: + if klass.__name__ == classname: + return True + return False + +def get_doc_object(obj, what=None, config=None): + if what is None: + if inspect.isclass(obj): + what = 'class' + elif inspect.ismodule(obj): + what = 'module' + elif callable(obj): + what = 'function' + else: + what = 'object' + if what == 'class': + doc = SphinxTraitsDoc(obj, '', func_doc=SphinxFunctionDoc, config=config) + if looks_like_issubclass(obj, 'HasTraits'): + for name, trait, comment in comment_eater.get_class_traits(obj): + # Exclude private traits. + if not name.startswith('_'): + doc['Traits'].append((name, trait, comment.splitlines())) + return doc + elif what in ('function', 'method'): + return SphinxFunctionDoc(obj, '', config=config) + else: + return SphinxDocString(pydoc.getdoc(obj), config=config) + +def setup(app): + # init numpydoc + numpydoc.setup(app, get_doc_object) + diff --git a/astropy_helpers/sphinx/ext/utils.py b/astropy_helpers/sphinx/ext/utils.py new file mode 100644 index 00000000..2a06c83b --- /dev/null +++ b/astropy_helpers/sphinx/ext/utils.py @@ -0,0 +1,65 @@ +import inspect +import sys + + +def find_mod_objs(modname, onlylocals=False): + """ Returns all the public attributes of a module referenced by name. + + .. note:: + The returned list *not* include subpackages or modules of + `modname`,nor does it include private attributes (those that + beginwith '_' or are not in `__all__`). + + Parameters + ---------- + modname : str + The name of the module to search. + onlylocals : bool + If True, only attributes that are either members of `modname` OR one of + its modules or subpackages will be included. + + Returns + ------- + localnames : list of str + A list of the names of the attributes as they are named in the + module `modname` . + fqnames : list of str + A list of the full qualified names of the attributes (e.g., + ``astropy.utils.misc.find_mod_objs``). For attributes that are + simple variables, this is based on the local name, but for + functions or classes it can be different if they are actually + defined elsewhere and just referenced in `modname`. + objs : list of objects + A list of the actual attributes themselves (in the same order as + the other arguments) + + """ + + __import__(modname) + mod = sys.modules[modname] + + if hasattr(mod, '__all__'): + pkgitems = [(k, mod.__dict__[k]) for k in mod.__all__] + else: + pkgitems = [(k, mod.__dict__[k]) for k in dir(mod) if k[0] != '_'] + + # filter out modules and pull the names and objs out + ismodule = inspect.ismodule + localnames = [k for k, v in pkgitems if not ismodule(v)] + objs = [v for k, v in pkgitems if not ismodule(v)] + + # fully qualified names can be determined from the object's module + fqnames = [] + for obj, lnm in zip(objs, localnames): + if hasattr(obj, '__module__') and hasattr(obj, '__name__'): + fqnames.append(obj.__module__ + '.' + obj.__name__) + else: + fqnames.append(modname + '.' + lnm) + + if onlylocals: + valids = [fqn.startswith(modname) for fqn in fqnames] + localnames = [e for i, e in enumerate(localnames) if valids[i]] + fqnames = [e for i, e in enumerate(fqnames) if valids[i]] + objs = [e for i, e in enumerate(objs) if valids[i]] + + return localnames, fqnames, objs diff --git a/astropy_helpers/sphinx/ext/viewcode.py b/astropy_helpers/sphinx/ext/viewcode.py new file mode 100644 index 00000000..45588db9 --- /dev/null +++ b/astropy_helpers/sphinx/ext/viewcode.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +""" + sphinx.ext.viewcode + ~~~~~~~~~~~~~~~~~~~ + + Add links to module code in Python object descriptions. + + :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. + + Patched using patch in https://bitbucket.org/birkenfeld/sphinx/issue/623/extension-viewcode-fails-with-function on 21 Aug 2013 by Kyle H Barbary +""" + +from docutils import nodes + +from sphinx import addnodes +from sphinx.locale import _ +from sphinx.pycode import ModuleAnalyzer +from sphinx.util.nodes import make_refnode + +import sys +import traceback + + +def doctree_read(app, doctree): + env = app.builder.env + if not hasattr(env, '_viewcode_modules'): + env._viewcode_modules = {} + + def get_full_modname(modname, attribute): + try: + __import__(modname) + except Exception, error: + if not app.quiet: + app.info(traceback.format_exc().rstrip()) + app.warn('viewcode can\'t import %s, failed with error "%s"' % + (modname, error)) + return None + module = sys.modules[modname] + try: + # Allow an attribute to have multiple parts and incidentially allow + # repeated .s in the attribute. + attr = attribute.split('.') + value = module + for attr in attribute.split('.'): + if attr: + value = getattr(value, attr) + except AttributeError: + app.warn('Didn\'t find %s in %s' % (attribute, module.__name__)) + return None + else: + return getattr(value, '__module__', None) + + + def has_tag(modname, fullname, docname, refname): + entry = env._viewcode_modules.get(modname, None) + if entry is None: + try: + analyzer = ModuleAnalyzer.for_module(modname) + except Exception: + env._viewcode_modules[modname] = False + return + analyzer.find_tags() + if not isinstance(analyzer.code, unicode): + code = analyzer.code.decode(analyzer.encoding) + else: + code = analyzer.code + entry = code, analyzer.tags, {}, refname + env._viewcode_modules[modname] = entry + elif entry is False: + return + _, tags, used, _ = entry + if fullname in tags: + used[fullname] = docname + return True + + + for objnode in doctree.traverse(addnodes.desc): + if objnode.get('domain') != 'py': + continue + names = set() + for signode in objnode: + if not isinstance(signode, addnodes.desc_signature): + continue + modname = signode.get('module') + fullname = signode.get('fullname') + refname = modname + if env.config.viewcode_import: + modname = get_full_modname(modname, fullname) + if not modname: + continue + if not has_tag(modname, fullname, env.docname, refname): + continue + if fullname in names: + # only one link per name, please + continue + names.add(fullname) + pagename = '_modules/' + modname.replace('.', '/') + onlynode = addnodes.only(expr='html') + onlynode += addnodes.pending_xref( + '', reftype='viewcode', refdomain='std', refexplicit=False, + reftarget=pagename, refid=fullname, + refdoc=env.docname) + onlynode[0] += nodes.inline('', _('[source]'), + classes=['viewcode-link']) + signode += onlynode + + +def missing_reference(app, env, node, contnode): + # resolve our "viewcode" reference nodes -- they need special treatment + if node['reftype'] == 'viewcode': + return make_refnode(app.builder, node['refdoc'], node['reftarget'], + node['refid'], contnode) + + +def collect_pages(app): + env = app.builder.env + if not hasattr(env, '_viewcode_modules'): + return + highlighter = app.builder.highlighter + urito = app.builder.get_relative_uri + + modnames = set(env._viewcode_modules) + + app.builder.info(' (%d module code pages)' % + len(env._viewcode_modules), nonl=1) + + for modname, entry in env._viewcode_modules.iteritems(): + if not entry: + continue + code, tags, used, refname = entry + # construct a page name for the highlighted source + pagename = '_modules/' + modname.replace('.', '/') + # highlight the source using the builder's highlighter + highlighted = highlighter.highlight_block(code, 'python', linenos=False) + # split the code into lines + lines = highlighted.splitlines() + # split off wrap markup from the first line of the actual code + before, after = lines[0].split('
')
+        lines[0:1] = [before + '
', after]
+        # nothing to do for the last line; it always starts with 
 anyway
+        # now that we have code lines (starting at index 1), insert anchors for
+        # the collected tags (HACK: this only works if the tag boundaries are
+        # properly nested!)
+        maxindex = len(lines) - 1
+        for name, docname in used.iteritems():
+            type, start, end = tags[name]
+            backlink = urito(pagename, docname) + '#' + refname + '.' + name
+            lines[start] = (
+                '
%s' % (name, backlink, _('[docs]'))
+                + lines[start])
+            lines[min(end - 1, maxindex)] += '
'
+        # try to find parents (for submodules)
+        parents = []
+        parent = modname
+        while '.' in parent:
+            parent = parent.rsplit('.', 1)[0]
+            if parent in modnames:
+                parents.append({
+                    'link': urito(pagename, '_modules/' +
+                                  parent.replace('.', '/')),
+                    'title': parent})
+        parents.append({'link': urito(pagename, '_modules/index'),
+                        'title': _('Module code')})
+        parents.reverse()
+        # putting it all together
+        context = {
+            'parents': parents,
+            'title': modname,
+            'body': _('
Source code for %s
') % modname + \
+                    '\n'.join(lines)
+        }
+        yield (pagename, context, 'page.html')
+
+    if not modnames:
+        return
+
+    app.builder.info(' _modules/index')
+    html = ['\n']
+    # the stack logic is needed for using nested lists for submodules
+    stack = ['']
+    for modname in sorted(modnames):
+        if modname.startswith(stack[-1]):
+            stack.append(modname + '.')
+            html.append('
    ')
+        else:
+            stack.pop()
+            while not modname.startswith(stack[-1]):
+                stack.pop()
+                html.append('
')
+            stack.append(modname + '.')
+        html.append('
  • %s
    • \n' % (
+            urito('_modules/index', '_modules/' + modname.replace('.', '/')),
+            modname))
+    html.append('' * (len(stack) - 1))
+    context = {
+        'title': _('Overview: module code'),
+        'body': _('
    All modules for which code is available
    ') + \
+            ''.join(html),
+    }
+
+    yield ('_modules/index', context, 'page.html')
+
+
+def setup(app):
+    app.add_config_value('viewcode_import', True, False)
+    app.connect('doctree-read', doctree_read)
+    app.connect('html-collect-pages', collect_pages)
+    app.connect('missing-reference', missing_reference)
+    #app.add_config_value('viewcode_include_modules', [], 'env')
+    #app.add_config_value('viewcode_exclude_modules', [], 'env')
diff --git a/astropy_helpers/sphinx/setup_package.py b/astropy_helpers/sphinx/setup_package.py
new file mode 100644
index 00000000..9e11a78f
--- /dev/null
+++ b/astropy_helpers/sphinx/setup_package.py
@@ -0,0 +1,9 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+def get_package_data():
+    # Install the theme files
+    return {
+        'astropy_helpers.sphinx': [
+            'ext/templates/*/*',
+            'themes/bootstrap-astropy/*.*',
+            'themes/bootstrap-astropy/static/*.*']}
diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/globaltoc.html b/astropy_helpers/sphinx/themes/bootstrap-astropy/globaltoc.html
new file mode 100644
index 00000000..3bd8404a
--- /dev/null
+++ b/astropy_helpers/sphinx/themes/bootstrap-astropy/globaltoc.html
@@ -0,0 +1,3 @@
+
    Table of Contents
    
+{{ toctree(maxdepth=-1, titles_only=true) }}
+
diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/layout.html b/astropy_helpers/sphinx/themes/bootstrap-astropy/layout.html
new file mode 100644
index 00000000..99d61963
--- /dev/null
+++ b/astropy_helpers/sphinx/themes/bootstrap-astropy/layout.html
@@ -0,0 +1,94 @@
+{% extends "basic/layout.html" %}
+
+{# Collapsible sidebar script from default/layout.html in Sphinx #}
+{% set script_files = script_files + ['_static/sidebar.js'] %}
+
+{# Add the google webfonts needed for the logo #}
+{% block extrahead %}
+
+{% endblock %}
+
+
+{% block header %}
+
    
+  {{ theme_logotext1 }}{{ theme_logotext2 }}{{ theme_logotext3 }}
+  
      
+    
      • 
+    
    • Index
      • 
+    
    • Modules
      • 
+    
    • 
+      {% block sidebarsearch %}
+      {% include "searchbox.html" %}
+      {% endblock %}
+    
      • 
+  
    
+
    
+{% endblock %}
+
+{% block relbar1 %}
+
    
+    
    {{ _('Navigation') }}
    
+    
      
+      {%- if next %}
+      
    • 
+	
+	  next {{ "»"|safe }}
+	
+      
      • 
+      {%- endif %}
+      {%- if prev %}
+      
    • 
+	
+	  {{ "«"|safe }} previous
+	
+	{% if next %}{{ reldelim2 }}{% endif %}
+      
      • 
+      {%- endif %}
+      {%- block rootrellink %}
+      
    • 
+	{{ shorttitle|e }}
+	{{ reldelim1 }}
+      
      • 
+      {%- endblock %}
+      {%- for parent in parents %}
+      
    • {{ parent.title }}{{ reldelim1 }}
      • 
+      {%- endfor %}
+      {# Don't put the title in the relbar for the first (index) page. #}
+      {% if prev %}
    • {{ title }}
      • {% endif %}
+      {%- block relbaritems %} {% endblock %}
+    
    
+
    
+{% endblock %}
+
+{# Silence the bottom relbar. #}
+{% block relbar2 %}{% endblock %}
+
+
+{%- block footer %}
+
    
+  
    
+    {%- if edit_on_github %}
+    {{ edit_on_github_page_message }}  
+    {%- endif %}
+    {%- if show_source and has_source and sourcename %}
+    {{ _('Page Source') }}
+    {%- endif %}  
+    Back to Top
    
+  
    
+    {%- if show_copyright %}
+    {%- if hasdoc('copyright') %}
+    {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    
+    {%- else %}
+    {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    
+    {%- endif %}
+    {%- endif %}
+    {%- if show_sphinx %}
+    {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %}  
+    {%- endif %}
+    {%- if last_updated %}
+    {% trans last_updated=last_updated|e %}Last built {{ last_updated }}.{% endtrans %} 
    
+    {%- endif %}
+  
    
+
    
+{%- endblock %}
diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/localtoc.html b/astropy_helpers/sphinx/themes/bootstrap-astropy/localtoc.html
new file mode 100644
index 00000000..0a21ad00
--- /dev/null
+++ b/astropy_helpers/sphinx/themes/bootstrap-astropy/localtoc.html
@@ -0,0 +1,3 @@
+
    Page Contents
    
+{{ toc }}
+
diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/searchbox.html b/astropy_helpers/sphinx/themes/bootstrap-astropy/searchbox.html
new file mode 100644
index 00000000..0d17cc14
--- /dev/null
+++ b/astropy_helpers/sphinx/themes/bootstrap-astropy/searchbox.html
@@ -0,0 +1,7 @@
+{%- if pagename != "search" %}
+
    
+  
+  
+  
+
    
+{%- endif %}
diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_linkout_20.png b/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_linkout_20.png
new file mode 100644
index 0000000000000000000000000000000000000000..432267972c8fd559ad75cf869713e3dea797a684
GIT binary patch
literal 1725
zcmZ{kX*8Q@8^@o>peY(Us9L7BRH-GXh>$797AiWZ+BzOfkS3`Rue76-8BANc1+~OJ
z!PJymBdVrPm$6N2D-}|jE||OoQ`6`O@}?i(4|DGGyMO2W&-Fjox9iGDqPN;!-Ms(+
zs1bZT0_Bd7qq#>({=db#8Oo&;4-a{O2%@?v@~Tq#RAa60nr|>4+KS&-3ZrT?TwsNtW#qhZwMQ3F2QR@~
zEJbsyehI7}uI74l9fJ;<80AWV0hqUP-wpAVzqV^R2WUwu6^eInrIv_yArxc?ayiGg
zClwHMoiXK5B^L!t)}(Di4h}31%uIfKQV7-@ShsFPQ#2~Z^nN+*D=J_-=DJn}er+z=
z9DzECsa2+;AP_^x(1ZDhfSY1k-A%wDw?qc?`;-?lu-c_s21D=Cg|1+)`a{)CU)*#W
zejNpZHbN(GNmyY+E*ED)k)SN5Iy?Qxr3xQMz|Z~tTvnrTe_?h~R(yRYuGXeiGcj;@
zcz>Ijf|_qxq_G4Po)f;Z*oiQ@^IWn36>2ZMX8nI8hJt_z1T~jnYoOX7z3o@Ci2-EuX(*o%70JfO1tj
z4F&vlF=7RaDaqkPg&luoB@+E0=J90i7}^!=v;xy3nKC<_yLWb2{|G6p)4_E3o-#8o
z`+MVc57fgFa=3qDd#A0Uj*V!%DMqkLlZFebZ!Jt{^3VgzT9s$i3g96utLsR3)XZ;A
zlBwk%+jE<2SN%}U=93EtTHNN=U${%}186O5GvriEg{`Vuw26sap&6@6)gNgy!q*iD
zU`eoPn6pAR+#87X#S=#1PqNurhq87d62#_8TN;H^xL^$HRb}3X48@m=!yQ>aSDaK4
z=6KaOw(<%wz8QX7s)?4*#-eM3egZ}cu7-ddPClv+*bY5~5JeD5;Jv_UPxWh%qU`cy
ztIMiCH{Y4_sAq{Zm)K!DdV2V*5LIKhXF=!HY^N_85VUEAI=&yAvj|~2!gtfU!gSK0aE#_ooPsJtt~!jy+C`rlzI)Zks058elxHQJ5@zX?>L+
zm!5LqymCLY_a
zkzc4PhQL%z{ci*<-*HPnHQ`Mf_qu=fu(L#B!X_XI;Rzb~U&i@etl7xI4=MG-eZzwN
zHO#1<++JNy(e07R{EnxhGuuaidp=t=dfWc+@)q0x2p
zsQCXY(30ZF@fZHR@WN_oR$e&xc|%ZAd@3_Kg#<8}Ol$Hv3L`d}PO?r)Nno$K=*oux
Mf+x|V#U1?Ne-+Xj00000

literal 0
HcmV?d00001

diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo.ico b/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo.ico
new file mode 100644
index 0000000000000000000000000000000000000000..c2bca3af116dc3c3f50313c7f5620c74ff255837
GIT binary patch
literal 1150
zcmYjRZ)lZe6#u>N-Q7R6E-U8TgkAN{l{Nb_i%5*+3N>Sn)tb~mf_+HvGNNs1lDp29
zTQzrwnKJfajXsPOA|XSVAO&VaaKjc8BiJfGlx!umd3Wz~ey4Nq{nGpLd(LzI{GM~3
z^E~ehc=m=uurZ8xa)4q0j~L8kmaN?bV7}QkS!DF|5E0~5iU&BuiW%IN`~nwU{F|p+
zqH@CY#W24PC7%GgrQM}Mo4xy)|6sz4>;W|kT3{vgkwwHx?z-jJ!
zSqzrRnLwG81j`(}N(o>qfht3V!|G~RjuJOt{;cTSAC#p8-L7UyA?Q+G$O+asqR6Qq
zb_>DHZg;_>g23ap#t!2n1yss-?qj0A{>H?$pUAvTqInyHhq{HM7s;V9%HD&Lw3_Kf
zL;_Dpea=>aUnPMDYglVN(D&n%`$=+DwJ`h=_0C;tGNt<64eIo5W;S$nnyzR&6;*OC
zZwD(-P53Xc2irt%&C{LvnoO@Xb(=alOZCley~-fmK+gX~y?T>L*gmnHqp%vVQ$}-N
zp(c`a4V~n}uV+%}lW$Y^9Hw5j{6m+hg*!#VjhQ;w)MYh1C(9OFg;!-br-8b(Lbtz{
zEIB}3PEapi&#selSvfReoM%L@Op_NT>F}ax{%dRtHnR=;M1TB+?);?HUn@FsGs{UI
z;`kKx%kQlRbH=wC&Rrs>=8Rt_8hDe#MNz9^zbx0qGaT;ivo)GTo9qtk>Nifa=*;ib
zksoZnNA&!4^2)67-w+M9SPxq59HJ&$L_e8eGM#he8d-CSTK5HY<_>jXiCWffe2e+#
zK6&JV@ekM;w_5Mp?GC)h3csEA5EX40q5JtyYKr8FSRZatOFOCSj#4M=9Pj>179SJA
zyOzI$wcb$%{Nsf8A?y7Z1szs@2a^R|l-wBUwVB>ZelwHZU*}~e&_)f$IF3FA4oaFE
z@Oo_Ct-Axgy5RMy;$x-hQuvvs>c%v6_6FTs-%|2F&G_d1ZkC%LIAV9;up!0(`b|H@
z67=(PZ-9#!ussiu&)dCPmG0$9O`?xuh_e)f7JHH&j%V@$ADKL%K6=%|X`7$ZVw}}R
zLkw~4$sg21YroPh_`rO}=vjUbT|_(r)D&9!puxA#$^HTvnE3w_GbN20^A8*9EjCgB
F{0C|Ra})po

literal 0
HcmV?d00001

diff --git a/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo_32.png b/astropy_helpers/sphinx/themes/bootstrap-astropy/static/astropy_logo_32.png
new file mode 100644
index 0000000000000000000000000000000000000000..fc3d93099fdfc3bfb09fb4cb9e26610a918fd874
GIT binary patch
literal 1884
zcmV-i2c!6jP)9Klkxtkx;N8hyx0QMuSyC?8ulV#bjxe
zNh1ldozlcgXhSmQ$BZ@6)O2?`?c1s~(b5@H(#EQ(>4e&$C8&*B$wo4bR%;TiN=5{X
zV8MzgN@Ush_TBz*Vc#w+49P#eGiT<#bI*_O{hs^Io)x;T^Z#7YU&P6k?9L;a#Yl>Q
zY#h*_wgh~;4ssuW
z-WF&-2uC(Tf0u0{`H@HJ)x)n5OW?wtgM}B9oG`j0(HMh|92T
zOg6Z1&l(eeE8xzQgLtLEbZJ0DF~<-Ckz
zfCVjHJ63?Kg?{flO%CV9@Jo1|RS51Ks+C&#OA#eHOpW~IiJT(1>pA3}7vWUBa^35O
zGY1e4S3~+&Vc{A$`UmJa7xc4W8E`aM0#~;?OPY8>Y6bJF5X*Pj$)$wW_u-TM(Aoso
zhb~v@Ohn~h5oI;XcVH0C97OJVM(syC5Z`(^?7j#;k3h(0Xh~iW296-a;#G*Ho7A)6
zgT_tBe{6*-=k4Ri#DQ}qV)0stON6_hg2Qh>`w_)p^C3978}gUHtZ#xT)FIT$aCNy8
zGyqqdJ5?C`Fo-bMX(#vHKvtI^>s-)%KK$0-^ueAD$aM?A^n(zH@4gDA7W`sl>k$>l
zkpPKV;r>2ywV`^KV=!shNi1A3M2m^sSPZ8RDTmDY2-k;**ZhcIUxkWoAY%dycQqGJ(N
zHTJt<*-3H=ZT&EtO8S6qjX_evPz3_Y_;QD8ttOB)58{&)E8d;(=k=pOOd2}rwM+0p
zgUwv-5CH!Gw4MrO;E*R@4~z!jyQv)F5^S9oXD9NT$ZannH+&ELzHksXdXcYGK;Jdx
zmoOCyOV!@FkHBY!_JM<-`amycNRWXM002&W0N1ZTVhUox!^rv?I9Q8p{1YsB8fJYB
zy4s+w8V356j}VAeJ0K}lF}H8KVpFw&0SsytfPUJ-rh;JrQ!*hYK?RfnIJ_4YFNcEr
zAio%nH>zlLuZ}JUK>719zeGL1Uk`6@4Q@&J{Vx`&2}$#m=r=Iv8m2}b-lwc4CBXd^
z!L}?9)GB>+4C0r6gL}&(|0E~RK>TzUV#UkKzUM#4bx(xa3~DC9p+xAq&PxsMEdo^I
zj)_Gydc^R6xTHWnH%o`QJ;3jk_Q&9d&=gBu7vSVSk^0l{yLE8wYUoVysSqsjJgfJGA_`tVVNzwm
z3BW;5FRWRP*!-3PvEh$MOGEB`OBpx+8`->8Jtt0vv<&EI1K&U-E&w_-!F`@`y>sM7
za3X5TcV79%ac@5sC5nY+_~ay_*82(JEUaI`e*gpu?!+IFuc~U(6?(A
zw_)%)1_rfnV%;+bxpj*&aKs@VeRA};Zqh6RRBGMiP9!-@;J=}3lY4T&;z3PNDq*J9
za%nE9X`0+y@0ZKU&IxFf(5RX0_q?b#PbBH*0x$}Y%au%LrzA7;GmppP|GWuDuKxip
W!4|zZJm1Fv0000