diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a4ede411..2974bc2d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,6 +163,16 @@ jobs: - TEST_WHEELINSTALL_ENV: "wheelinstall-py37" <<: *ci-steps + python35-min-req: + docker: + - image: circleci/python:3.5.9-stretch + environment: + - TEST_TOX_ENV: "py35-min-req" + - COVERAGE_TOX_ENV: "" + - BUILD_TOX_ENV: "build-py35-min-req" + - TEST_WHEELINSTALL_ENV: "wheelinstall-py35-min-req" + <<: *ci-steps + miniconda35: docker: - image: continuumio/miniconda3:4.3.27p0 @@ -207,6 +217,13 @@ jobs: - TEST_TOX_ENV: "gallery-py37" <<: *gallery-steps + gallery36-min-req: + docker: + - image: circleci/python:3.6.3-jessie + environment: + - TEST_TOX_ENV: "gallery-py36-min-req" + <<: *gallery-steps + deploy-dev: docker: - image: circleci/python:3.7.6-stretch @@ -266,6 +283,8 @@ workflows: <<: *no_filters - python37: <<: *no_filters + - python35-min-req: + <<: *no_filters - miniconda35: <<: *no_filters - miniconda36: @@ -276,17 +295,21 @@ workflows: <<: *no_filters - gallery37: <<: *no_filters + - gallery36-min-req: + <<: *no_filters - deploy-dev: requires: - flake8 - python35 - python36 - python37 + - python35-min-req - miniconda35 - miniconda36 - miniconda37 - gallery36 - gallery37 + - gallery36-min-req filters: tags: ignore: @@ -301,11 +324,13 @@ workflows: - python35 - python36 - python37 + - python35-min-req - miniconda35 - miniconda36 - miniconda37 - gallery36 - gallery37 + - gallery36-min-req filters: tags: only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2fdbaacf..3d17c241b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,6 +29,14 @@ jobs: buildToxEnv: 'build-py35' testWheelInstallEnv: 'wheelinstall-py35' + macOS-py3.5-min-req: + imageName: 'macos-10.13' + pythonVersion: '3.5' + testToxEnv: 'py35-min-req' + coverageToxEnv: '' + buildToxEnv: 'build-py35-min-req' + testWheelInstallEnv: 'wheelinstall-py35-min-req' + Windows-py3.7: imageName: 'vs2017-win2016' pythonVersion: '3.7' @@ -53,6 +61,14 @@ jobs: buildToxEnv: 'build-py35' testWheelInstallEnv: 'wheelinstall-py35' + Windows-py3.5-min-req: + imageName: 'vs2017-win2016' + pythonVersion: '3.5' + testToxEnv: 'py35-min-req' + coverageToxEnv: '' + buildToxEnv: 'build-py35-min-req' + testWheelInstallEnv: 'wheelinstall-py35-min-req' + pool: vmImage: $(imageName) diff --git a/docs/source/software_process.rst b/docs/source/software_process.rst index 27e34ee3b..3934b450f 100644 --- a/docs/source/software_process.rst +++ b/docs/source/software_process.rst @@ -8,17 +8,20 @@ Software Process Continuous Integration ---------------------- -PyNWB is tested against Ubuntu, macOS and Windows operating systems. +PyNWB is tested against Ubuntu, macOS, and Windows operating systems. The project has both unit and integration tests. -* CircleCI runs all PyNWB tests on Ubuntu -* Appveyor runs all PyNWB tests on Windows -* Travis runs all PyNWB tests on macOS +* CircleCI_ runs all PyNWB tests on Ubuntu +* `Azure Pipelines`_ runs all PyNWB tests on Windows and macOS -Each time a PR is published or updated, the project is built, packaged and tested on all support operating systems and python distributions. That way, as a contributor you know if you introduced regressions or coding style inconsistencies. +Each time a PR is published or updated, the project is built, packaged, and tested on all supported operating systems +and python distributions. That way, as a contributor, you know if you introduced regressions or coding style +inconsistencies. There are badges in the README_ file which shows the current condition of the dev branch. +.. _CircleCI: https://circleci.com/gh/NeurodataWithoutBorders/workflows/pynwb +.. _Azure Pipelines: https://dev.azure.com/NeurodataWithoutBorders/pynwb/_build .. _README: https://github.com/NeurodataWithoutBorders/pynwb#readme @@ -40,27 +43,38 @@ lines are covered by the tests. Requirement Specifications -------------------------- -There are 2 kinds of requirements specification in PyNWB. +There are 4 kinds of requirements specification in PyNWB. Setup.py Dependencies --------------------- -The first one is the dependencies_ in the `setup.py` file which lists the abstract dependencies for -the PyNWB project. Note that there should not be specific versions of packages in the `setup.py` file. +There are 4 kinds of requirements specification in PyNWB. -Requirements.txt Dependencies ------------------------------ +The first one is the requirements-min.txt_ file, which lists the package dependencies and their minimum versions for +installing PyNWB. These dependencies are read by setup.py_ into the `install_requires` key, with the adjustment that +the `'=='` listed in `requirements-min.txt` are replaced with `'>='` to reflect that they are minimum versions. -The second one is `requirements.txt` which contain a list of pinned (concrete) dependencies to reproduce -an entire development environment to work with PyNWB. +The second one is requirements.txt_ which contain a list of pinned (concrete) dependencies to reproduce +an entire development environment to use PyNWB. -In order to check the status of the required packages requires.io_ is used to create a badge on the project -README_. If all the required packages are up to date, -a green badge appears. +The third one is requirements-dev.txt_ which contain a list of pinned (concrete) dependencies to reproduce +an entire development environment to use PyNWB, run PyNWB tests, check code style, compute coverage, and create test +environments. + +The final one is requirements-doc.txt_ which contain a list of dependencies to generate the documentation for PyNWB. +Both this file and `requirements.txt` are used by ReadTheDocs_ to initialize the local environment for Sphinx to run. + +In order to check the status of the required packages, requires.io_ is used to create a badge on the project +README_. If all the required packages are up to date, a green badge appears. If some of the packages are outdated, see :ref:`update_requirements_files`. -.. _dependencies: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/setup.py +.. _requirements-min.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-min.txt +.. _setup.py: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/setup.py +.. _requirements.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements.txt +.. _requirements-dev.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-dev.txt +.. _requirements-doc.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-doc.txt +.. _ReadTheDocs: https://readthedocs.org/projects/pynwb/ .. _requires.io: https://requires.io/github/NeurodataWithoutBorders/pynwb/requirements/?branch=dev @@ -69,10 +83,16 @@ Versioning and Releasing ------------------------- PyNWB uses versioneer_ for versioning source and wheel distributions. Versioneer creates a semi-unique release -name for the wheels that are created. It requires a version control system (git in PyNWB's case) to generate a release name. -After all the tests pass, CircleCI creates both wheels(*.whl) and source distribution(*.tgz) for Python 3 -and uploads them back to GitHub as a release_. Versioneer makes it possible to get the source distribution from GitHub and create -wheels directly without having to use a version control system because it hardcodes versions in the source distribution. +name for the wheels that are created. It requires a version control system (git in PyNWB's case) to generate a release +name. After all the tests pass, CircleCI creates both a wheel (\*.whl) and source distribution (\*.tar.gz) for Python 3 +and uploads them back to GitHub as a release_. Versioneer makes it possible to get the source distribution from GitHub +and create wheels directly without having to use a version control system because it hardcodes versions in the source +distribution. + +It is important to note that GitHub automatically generates source code archives in .zip and .tar.gz formats and +attaches those files to all releases as an asset. These files currently do not contain the submodules within PyNWB and +thus do not serve as a complete installation. For a complete source code archive, use the source distribution generated +by CircleCI, typically named `pynwb-{version}.tar.gz`. .. _versioneer: https://github.com/warner/python-versioneer .. _release: https://github.com/NeurodataWithoutBorders/pynwb/releases diff --git a/docs/source/update_requirements.rst b/docs/source/update_requirements.rst index 50ca1b6e4..991c74712 100644 --- a/docs/source/update_requirements.rst +++ b/docs/source/update_requirements.rst @@ -10,15 +10,17 @@ The different requirements files introduced in :ref:`software_process` section a * requirements.txt_ * requirements-dev.txt_ * requirements-doc.txt_ +* requirements-min.txt_ .. _requirements.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements.txt .. _requirements-dev.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-dev.txt .. _requirements-doc.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-doc.txt +.. _requirements-min.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-min.txt requirements.txt ================ -`Requirements.txt` of the project can be created or updated and then captured using +`requirements.txt` of the project can be created or updated and then captured using the following script: .. code:: @@ -63,3 +65,12 @@ the following scripts: deactivate rmvirtualenv pynwb-requirements + + +requirements-min.txt +==================== + +Minimum requirements should be updated manually if a new feature or bug fix is added in a dependency that is required +for proper running of PyNWB. Minimum requirements should also be updated if a user requests that PyNWB be installable +with an older version of a dependency, all tests pass using the older version, and there is no valid reason for the +minimum version to be as high as it is. diff --git a/requirements-dev.txt b/requirements-dev.txt index 122989cc7..0eed7dbcb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ codecov==2.0.15 -coverage==4.5.4 -flake8==3.7.8 -tox==3.14.0 --r requirements.txt +coverage==5.0.3 +flake8==3.7.9 +tox==3.14.3 diff --git a/requirements-doc.txt b/requirements-doc.txt index 8dee1a3be..a115e5dd5 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -2,6 +2,4 @@ sphinx matplotlib sphinx_rtd_theme sphinx-gallery -Glymur==0.8.19 # required for allensdk 1.3.0 to work -allensdk==1.3.0 --r requirements.txt +allensdk==1.3.2 diff --git a/requirements-min.txt b/requirements-min.txt new file mode 100644 index 000000000..74419aeba --- /dev/null +++ b/requirements-min.txt @@ -0,0 +1,6 @@ +# these minimum requirements specify '==' for testing; setup.py replaces '==' with '>=' +h5py==2.9 # support for setting attrs to lists of utf-8 added in 2.9 +hdmf==1.5.3,<2 +numpy==1.16 +pandas==0.23 +python-dateutil==2.7 diff --git a/setup.py b/setup.py index d0b523bfa..fbd089a60 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from setuptools import setup, find_packages -import re import versioneer @@ -13,10 +12,10 @@ schema_dir = 'nwb-schema/core' -# copy requirements from requirements.txt, ignore all pinned version info, but keep pinned version for hdmf -reqs_re = re.compile("[<=>]+") -with open('requirements.txt', 'r') as fp: - reqs = [reqs_re.split(x.strip())[0] if not x.startswith('hdmf') else x.strip() for x in fp.readlines()] +with open('requirements-min.txt', 'r') as fp: + # replace == with >= and remove trailing comments and spaces + reqs = [x.replace('==', '>=').split('#')[0].strip() for x in fp] + reqs = [x for x in reqs if x] # remove empty strings print(reqs) diff --git a/tox.ini b/tox.ini index b4eca5230..c1d16a8fa 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,14 @@ commands = coverage run test.py --pynwb coverage html -d tests/coverage/htmlcov +# Test with python 3.5, pinned dev reqs, and minimum run requirements +[testenv:py35-min-req] +basepython = python3.5 +deps = + -rrequirements-dev.txt + -rrequirements-min.txt +commands = {[testenv]commands} + # Envs that builds wheels and source distribution [testenv:build] commands = @@ -45,6 +53,13 @@ commands = {[testenv:build]commands} basepython = python3.7 commands = {[testenv:build]commands} +[testenv:build-py35-min-req] +basepython = python3.5 +deps = + -rrequirements-dev.txt + -rrequirements-min.txt +commands = {[testenv:build]commands} + # Envs that will only be executed on CI that does coverage reporting [testenv:coverage] passenv = CODECOV_TOKEN @@ -67,6 +82,10 @@ commands = python -c "import pynwb" deps = null commands = python -c "import pynwb" +[testenv:wheelinstall-py35-min-req] +deps = null +commands = python -c "import pynwb" + # Envs that will execute gallery tests [testenv:gallery] install_command = @@ -89,3 +108,11 @@ commands = {[testenv:gallery]commands} basepython = python3.7 deps = {[testenv:gallery]deps} commands = {[testenv:gallery]commands} + +[testenv:gallery-py36-min-req] +basepython = python3.6 +deps = + -rrequirements-dev.txt + -rrequirements-min.txt + -rrequirements-doc.txt +commands = {[testenv:gallery]commands}