From eb4be99ca5b1b257386f1dcb6ef8fdef89d004d9 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Fri, 11 Dec 2020 13:55:58 +0000 Subject: [PATCH] Only pin theano-pymc in requirements.txt (#4322) * :hammer: add scripts to check theano-pymc pins, to generate requirements-dev from conda, to sort conda deps * :wrench: update conda envs and requirements-dev.txt * :pencil: fix typo * :memo: add docstrings * :memo: update pip-to-conda docstring * noop * :fire: remove pin theano checks, just use requirements.txt * local install in arviz compat * :memo: switch -> checkout, note beginner friendly label * :memo: link to beginner friendly issues * run generate pip deps from conda on all conda env files * :wrench: make sure all conda-envs files generate requirements-dev * :twisted_rightwards_arrows: reorder some tests * :art: remove trailing comma * Update scripts/generate_pip_deps_from_conda.py Co-authored-by: Thomas Wiecki --- .github/workflows/arviz_compat.yml | 14 ++- .github/workflows/pytest.yml | 14 +++ .github/workflows/windows.yml | 17 +++ .pre-commit-config.yaml | 14 +++ CONTRIBUTING.md | 6 +- conda-envs/environment-dev-py36.yml | 47 ++++----- conda-envs/environment-dev-py37.yml | 43 +++----- conda-envs/environment-dev-py38.yml | 43 +++----- requirements-dev.txt | 32 +++--- scripts/generate_pip_deps_from_conda.py | 131 ++++++++++++++++++++++++ scripts/sort_conda_envs.py | 21 ++++ 11 files changed, 280 insertions(+), 102 deletions(-) create mode 100755 scripts/generate_pip_deps_from_conda.py create mode 100644 scripts/sort_conda_envs.py diff --git a/.github/workflows/arviz_compat.yml b/.github/workflows/arviz_compat.yml index 1e899568e07..34b4106f005 100644 --- a/.github/workflows/arviz_compat.yml +++ b/.github/workflows/arviz_compat.yml @@ -33,6 +33,18 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py38.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py38 @@ -47,7 +59,7 @@ jobs: - name: Install latest arviz run: | conda activate pymc3-dev-py38 - conda remove arviz -y + pip uninstall arviz -y pip install git+git://github.com/arviz-devs/arviz.git - name: Run tests run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2c16e9b6bbb..905128ca0f9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,6 +21,7 @@ jobs: --ignore=pymc3/tests/test_examples.py --ignore=pymc3/tests/test_gp.py --ignore=pymc3/tests/test_mixture.py + --ignore=pymc3/tests/test_ode.py --ignore=pymc3/tests/test_parallel_sampling.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_quadpotential.py @@ -43,6 +44,7 @@ jobs: - | pymc3/tests/test_examples.py pymc3/tests/test_mixture.py + pymc3/tests/test_ode.py pymc3/tests/test_posteriors.py pymc3/tests/test_quadpotential.py - | @@ -74,6 +76,18 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py36.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py36 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 6291cc80946..b5170fc876a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,12 +32,29 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py37.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py37 channel-priority: strict environment-file: conda-envs/environment-dev-py37.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + - name: Install-pymc3 + run: | + conda activate pymc3-dev-py37 + pip install -e . + python --version - run: | conda activate pymc3-dev-py37 python -m pytest -vv --cov=pymc3 --cov-report=xml --cov-report term --durations=50 $TEST_SUBSET diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9cb1b73b41..25d215c14dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: - id: no-commit-to-branch args: [--branch, master] - id: requirements-txt-fixer + exclude: ^requirements-dev\.txt$ - id: trailing-whitespace - repo: https://github.com/nbQA-dev/nbQA rev: 0.5.4 @@ -61,6 +62,19 @@ repos: language: python name: Check no tests are ignored pass_filenames: false + - id: conda-env-sort + additional_dependencies: [pyyaml] + entry: python scripts/sort_conda_envs.py + files: ^conda-envs/ + language: python + name: Sort dependencies in conda envs + types: [yaml] + - id: pip-from-conda + additional_dependencies: [pyyaml] + entry: python scripts/generate_pip_deps_from_conda.py + files: ^conda-envs/ + language: python + name: Generate pip dependency from conda - id: no-relative-imports name: No relative imports entry: from \.[\.\w]* import diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05173c1e975..2d56fc4e8a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,8 @@ We appreciate being notified of problems with the existing PyMC code. We prefer Please verify that your issue is not being currently addressed by other issues or pull requests by using the GitHub search tool to look for key words in the project issue tracker. +Filter on the ["beginner friendly"](https://github.com/pymc-devs/pymc3/issues?q=is%3Aopen+is%3Aissue+label%3A%22beginner+friendly%22) label for issues which are good for new contributors. + # Contributing code via pull requests While issue reporting is valuable, we strongly encourage users who are inclined to do so to submit patches for new or existing issues via pull requests. This is particularly the case for simple fixes, such as typos or tweaks to documentation, which do not require a heavy investment of time and attention. @@ -38,7 +40,7 @@ The preferred workflow for contributing to PyMC3 is to fork the [GitHub reposito 3. Create a ``feature`` branch to hold your development changes: ```bash - $ git switch -c my-feature + $ git checkout -b my-feature ``` Always use a ``feature`` branch. It's good practice to never routinely work on the ``master`` branch of any repository. @@ -147,4 +149,6 @@ Follow [TensorFlow's style guide](https://www.tensorflow.org/community/contribut For documentation strings, we *prefer* [numpy style](https://numpydoc.readthedocs.io/en/latest/format.html) to comply with the style that predominates in our upstream dependencies. +Finally, see the [PyMC3 Python Code Style](https://github.com/pymc-devs/pymc3/wiki/PyMC3-Python-Code-Style) and the [PyMC's Jupyter Notebook Style](https://github.com/pymc-devs/pymc3/wiki/PyMC's-Jupyter-Notebook-Style) guides. + #### This guide was derived from the [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md) diff --git a/conda-envs/environment-dev-py36.yml b/conda-envs/environment-dev-py36.yml index 31f7c970985..6f6bf11a887 100644 --- a/conda-envs/environment-dev-py36.yml +++ b/conda-envs/environment-dev-py36.yml @@ -1,31 +1,22 @@ name: pymc3-dev-py36 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.6 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - dataclasses # python_version < 3.7 - - contextvars # python_version < 3.7 - - mkl-service - - dill - - libblas=*=*mkl +- contextvars +- dataclasses +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.6 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 7c8ad623321..1a307dfd0f1 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -1,29 +1,20 @@ name: pymc3-dev-py37 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.7 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - mkl-service - - dill - - libblas=*=*mkl +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.7 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/conda-envs/environment-dev-py38.yml b/conda-envs/environment-dev-py38.yml index 1d2176edcfd..24e9040cb31 100644 --- a/conda-envs/environment-dev-py38.yml +++ b/conda-envs/environment-dev-py38.yml @@ -1,29 +1,20 @@ name: pymc3-dev-py38 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.8 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - mkl-service - - dill - - libblas=*=*mkl +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.8 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e127fe8b9e..1396c4ee183 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,14 @@ -bokeh>=0.12.13 -coverage>=5.1 -dill -graphviz>=0.8.3 -ipython -nbsphinx>=0.4.2 -nose>=1.3.7 -nose-parameterized==0.6.0 -numpydoc>=0.9.1 -parameterized +# This file is auto-generated from by scripts/generate_pip_deps_from_conda.py, do not modify. +# See that file for comments about the need/usage of each dependency. + +h5py>=2.7 +ipython>=7.16 +nbsphinx>=0.4 +numpydoc>=0.9 pre-commit>=2.8.0 -pycodestyle>=2.3.1 -pyflakes>=1.5.0 -pylint>=1.7.4 -pytest>=3.0.7 -pytest-cov>=2.5.1 -pytest-xdist -recommonmark>=0.4.0 -seaborn>=0.8.1 -sphinx>=1.5.5 -sphinx-autobuild==0.7.1 +pytest-cov>=2.5 +pytest>=3.0 +recommonmark>=0.4 +sphinx-autobuild>=0.7 +sphinx>=1.5 watermark diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py new file mode 100755 index 00000000000..04a764d57e3 --- /dev/null +++ b/scripts/generate_pip_deps_from_conda.py @@ -0,0 +1,131 @@ +# BSD 3-Clause License + +# Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +# All rights reserved. + +# Copyright (c) 2011-2020, Open source contributors. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Check requirements-dev.txt has been generated from conda-envs/environment-dev-py3*.yml + +This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`. +You can run it manually with `pre-commit run pip-from-conda --all`. +""" +import argparse +import re + +import yaml + +EXCLUDE = {"python", "libblas", "mkl-service", "python-graphviz"} +RENAME = {} + + +def conda_package_to_pip(package): + """ + Convert a conda package to its pip equivalent. + + In most cases they are the same, those are the exceptions: + - Packages that should be excluded (in `EXCLUDE`) + - Packages that should be renamed (in `RENAME`) + - A package requiring a specific version, in conda is defined with a single + equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``) + """ + package = re.sub("(?<=[^<>])=", "==", package).strip() + + for compare in ("<=", ">=", "=="): + if compare not in package: + continue + pkg, version = package.split(compare, maxsplit=1) + if pkg in EXCLUDE: + return + + if pkg in RENAME: + return "".join((RENAME[pkg], compare, version)) + + break + + if package in EXCLUDE: + return + + if package in RENAME: + return RENAME[package] + + return package + + +def main(conda_fname, pip_fname): + """ + Generate the pip dependencies file from the conda file, or compare that + they are synchronized (``compare=True``). + + Parameters + ---------- + conda_fname : str + Path to the conda file with dependencies (e.g. `environment.yml`). + pip_fname : str + Path to the pip file with dependencies (e.g. `requirements-dev.txt`). + compare : bool, default False + Whether to generate the pip file (``False``) or to compare if the + pip file has been generated with this script and the last version + of the conda file (``True``). + + Returns + ------- + bool + True if the comparison fails, False otherwise + """ + with open(conda_fname) as conda_fd: + deps = yaml.safe_load(conda_fd)["dependencies"] + + pip_deps = [] + for dep in deps: + if isinstance(dep, str): + conda_dep = conda_package_to_pip(dep) + if conda_dep: + pip_deps.append(conda_dep) + elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep: + pip_deps += dep["pip"] + else: + raise ValueError(f"Unexpected dependency {dep}") + + header = ( + f"# This file is auto-generated by scripts/generate_pip_deps_from_conda.py, " + "do not modify.\n# See that file for comments about the need/usage of each dependency.\n\n" + ) + pip_content = header + "\n".join(pip_deps) + "\n" + + with open(pip_fname, "w") as pip_fd: + pip_fd.write(pip_content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + args = parser.parse_args() + for file in args.files: + main(file, "requirements-dev.txt") diff --git a/scripts/sort_conda_envs.py b/scripts/sort_conda_envs.py new file mode 100644 index 00000000000..299892858e4 --- /dev/null +++ b/scripts/sort_conda_envs.py @@ -0,0 +1,21 @@ +""" +Sort dependencies in conda environment files. + +This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`. +You can run it manually with `pre-commit run conda-env-sort --all`. +""" + +import argparse + +import yaml + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + args = parser.parse_args() + for file_ in args.files: + with open(file_) as fd: + doc = yaml.safe_load(fd) + doc["dependencies"].sort() + with open(file_, "w") as fd: + yaml.dump(doc, fd, sort_keys=False)