Add and test minimum requirements for PyNWB

.circleci/config.yml

@@ -163,6 +163,16 @@ jobs:
      - TEST_WHEELINSTALL_ENV: "wheelinstall-py37"
     <<: *ci-steps
 
+  python35-min-req:
+    docker:
+      - image: circleci/python:3.5.9-stretch
+    environment:
+     - TEST_TOX_ENV: "py35-min-req"
+     - COVERAGE_TOX_ENV: ""
+     - BUILD_TOX_ENV: "build-py35-min-req"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall-py35-min-req"
+    <<: *ci-steps
+
   miniconda35:
     docker:
       - image: continuumio/miniconda3:4.3.27p0
@@ -207,6 +217,13 @@ jobs:
      - TEST_TOX_ENV: "gallery-py37"
     <<: *gallery-steps
 
+  gallery36-min-req:
+    docker:
+      - image: circleci/python:3.6.3-jessie
+    environment:
+     - TEST_TOX_ENV: "gallery-py36-min-req"
+    <<: *gallery-steps
+
   deploy-dev:
     docker:
       - image: circleci/python:3.7.6-stretch
@@ -266,6 +283,8 @@ workflows:
           <<: *no_filters
       - python37:
           <<: *no_filters
+      - python35-min-req:
+          <<: *no_filters
       - miniconda35:
           <<: *no_filters
       - miniconda36:
@@ -276,17 +295,21 @@ workflows:
           <<: *no_filters
       - gallery37:
           <<: *no_filters
+      - gallery36-min-req:
+          <<: *no_filters
       - deploy-dev:
           requires:
             - flake8
             - python35
             - python36
             - python37
+            - python35-min-req
             - miniconda35
             - miniconda36
             - miniconda37
             - gallery36
             - gallery37
+            - gallery36-min-req
           filters:
             tags:
               ignore:
@@ -301,11 +324,13 @@ workflows:
             - python35
             - python36
             - python37
+            - python35-min-req
             - miniconda35
             - miniconda36
             - miniconda37
             - gallery36
             - gallery37
+            - gallery36-min-req
           filters:
             tags:
               only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/

azure-pipelines.yml

@@ -29,6 +29,14 @@ jobs:
         buildToxEnv: 'build-py35'
         testWheelInstallEnv: 'wheelinstall-py35'
 
+      macOS-py3.5-min-req:
+        imageName: 'macos-10.13'
+        pythonVersion: '3.5'
+        testToxEnv: 'py35-min-req'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py35-min-req'
+        testWheelInstallEnv: 'wheelinstall-py35-min-req'
+
       Windows-py3.7:
         imageName: 'vs2017-win2016'
         pythonVersion: '3.7'
@@ -53,6 +61,14 @@ jobs:
         buildToxEnv: 'build-py35'
         testWheelInstallEnv: 'wheelinstall-py35'
 
+      Windows-py3.5-min-req:
+        imageName: 'vs2017-win2016'
+        pythonVersion: '3.5'
+        testToxEnv: 'py35-min-req'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py35-min-req'
+        testWheelInstallEnv: 'wheelinstall-py35-min-req'
+
   pool:
     vmImage: $(imageName)
 

docs/source/software_process.rst

@@ -8,17 +8,20 @@ Software Process
 Continuous Integration
 ----------------------
 
-PyNWB is tested against Ubuntu, macOS and Windows operating systems.
+PyNWB is tested against Ubuntu, macOS, and Windows operating systems.
 The project has both unit and integration tests.
 
-* CircleCI runs all PyNWB tests on Ubuntu
-* Appveyor runs all PyNWB tests on Windows
-* Travis runs all PyNWB tests on macOS
+* CircleCI_ runs all PyNWB tests on Ubuntu
+* `Azure Pipelines`_ runs all PyNWB tests on Windows and macOS
 
-Each time a PR is published or updated, the project is built, packaged and tested on all support operating systems and python distributions. That way, as a contributor you know if you introduced regressions or coding style inconsistencies.
+Each time a PR is published or updated, the project is built, packaged, and tested on all supported operating systems
+and python distributions. That way, as a contributor, you know if you introduced regressions or coding style
+inconsistencies.
 
 There are badges in the README_ file which shows the current condition of the dev branch.
 
+.. _CircleCI: https://circleci.com/gh/NeurodataWithoutBorders/workflows/pynwb
+.. _Azure Pipelines: https://dev.azure.com/NeurodataWithoutBorders/pynwb/_build
 .. _README: https://github.com/NeurodataWithoutBorders/pynwb#readme
 
 
@@ -40,27 +43,38 @@ lines are covered by the tests.
 Requirement Specifications
 --------------------------
 
-There are 2 kinds of requirements specification in PyNWB.
+There are 4 kinds of requirements specification in PyNWB.
 
 Setup.py Dependencies
 ---------------------
 
-The first one is the dependencies_ in the `setup.py` file which lists the abstract dependencies for
-the PyNWB project. Note that there should not be specific versions of packages in the `setup.py` file.
+There are 4 kinds of requirements specification in PyNWB.
 
-Requirements.txt Dependencies
------------------------------
+The first one is the requirements-min.txt_ file, which lists the package dependencies and their minimum versions for
+installing PyNWB. These dependencies are read by setup.py_ into the `install_requires` key, with the adjustment that
+the `'=='` listed in `requirements-min.txt` are replaced with `'>='` to reflect that they are minimum versions.
 
-The second one is `requirements.txt` which contain a list of pinned (concrete) dependencies to reproduce
-an entire development environment to work with PyNWB.
+The second one is requirements.txt_ which contain a list of pinned (concrete) dependencies to reproduce
+an entire development environment to use PyNWB.
 
-In order to check the status of the required packages requires.io_ is used to create a badge on the project
-README_. If all the required packages are up to date,
-a green badge appears.
+The third one is requirements-dev.txt_ which contain a list of pinned (concrete) dependencies to reproduce
+an entire development environment to use PyNWB, run PyNWB tests, check code style, compute coverage, and create test
+environments.
+
+The final one is requirements-doc.txt_ which contain a list of dependencies to generate the documentation for PyNWB.
+Both this file and `requirements.txt` are used by ReadTheDocs_ to initialize the local environment for Sphinx to run.
+
+In order to check the status of the required packages, requires.io_ is used to create a badge on the project
+README_. If all the required packages are up to date, a green badge appears.
 
 If some of the packages are outdated, see :ref:`update_requirements_files`.
 
-.. _dependencies: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/setup.py
+.. _requirements-min.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-min.txt
+.. _setup.py: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/setup.py
+.. _requirements.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements.txt
+.. _requirements-dev.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-dev.txt
+.. _requirements-doc.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-doc.txt
+.. _ReadTheDocs: https://readthedocs.org/projects/pynwb/
 .. _requires.io: https://requires.io/github/NeurodataWithoutBorders/pynwb/requirements/?branch=dev
 
 
@@ -69,10 +83,16 @@ Versioning and Releasing
 -------------------------
 
 PyNWB uses versioneer_ for versioning source and wheel distributions. Versioneer creates a semi-unique release
-name for the wheels that are created. It requires a version control system (git in PyNWB's case) to generate a release name.
-After all the tests pass, CircleCI creates both wheels(*.whl) and source distribution(*.tgz) for Python 3
-and uploads them back to GitHub as a release_. Versioneer makes it possible to get the source distribution from GitHub and create
-wheels directly without having to use a version control system because it hardcodes versions in the source distribution.
+name for the wheels that are created. It requires a version control system (git in PyNWB's case) to generate a release
+name. After all the tests pass, CircleCI creates both a wheel (\*.whl) and source distribution (\*.tar.gz) for Python 3
+and uploads them back to GitHub as a release_. Versioneer makes it possible to get the source distribution from GitHub
+and create wheels directly without having to use a version control system because it hardcodes versions in the source
+distribution.
+
+It is important to note that GitHub automatically generates source code archives in .zip and .tar.gz formats and
+attaches those files to all releases as an asset. These files currently do not contain the submodules within PyNWB and
+thus do not serve as a complete installation. For a complete source code archive, use the source distribution generated
+by CircleCI, typically named `pynwb-{version}.tar.gz`.
 
 .. _versioneer: https://github.com/warner/python-versioneer
 .. _release: https://github.com/NeurodataWithoutBorders/pynwb/releases

docs/source/update_requirements.rst

@@ -10,15 +10,17 @@ The different requirements files introduced in :ref:`software_process` section a
 * requirements.txt_
 * requirements-dev.txt_
 * requirements-doc.txt_
+* requirements-min.txt_
 
 .. _requirements.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements.txt
 .. _requirements-dev.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-dev.txt
 .. _requirements-doc.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-doc.txt
+.. _requirements-min.txt: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/requirements-min.txt
 
 requirements.txt
 ================
 
-`Requirements.txt` of the project can be created or updated and then captured using
+`requirements.txt` of the project can be created or updated and then captured using
 the following script:
 
 .. code::
@@ -63,3 +65,12 @@ the following scripts:
 
    deactivate
    rmvirtualenv pynwb-requirements
+
+
+requirements-min.txt
+====================
+
+Minimum requirements should be updated manually if a new feature or bug fix is added in a dependency that is required
+for proper running of PyNWB. Minimum requirements should also be updated if a user requests that PyNWB be installable
+with an older version of a dependency, all tests pass using the older version, and there is no valid reason for the
+minimum version to be as high as it is.

requirements-dev.txt

@@ -1,5 +1,4 @@
 codecov==2.0.15
-coverage==4.5.4
-flake8==3.7.8
-tox==3.14.0
--r requirements.txt
+coverage==5.0.3
+flake8==3.7.9
+tox==3.14.3

requirements-doc.txt

@@ -2,6 +2,4 @@ sphinx
 matplotlib
 sphinx_rtd_theme
 sphinx-gallery
-Glymur==0.8.19  # required for allensdk 1.3.0 to work
-allensdk==1.3.0
--r requirements.txt
+allensdk==1.3.2

requirements-min.txt

@@ -0,0 +1,6 @@
+# these minimum requirements specify '==' for testing; setup.py replaces '==' with '>='
+h5py==2.9  # support for setting attrs to lists of utf-8 added in 2.9
+hdmf==1.5.3,<2
+numpy==1.16
+pandas==0.23
+python-dateutil==2.7

setup.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 from setuptools import setup, find_packages
-import re
 
 import versioneer
 
@@ -13,10 +12,10 @@
 
 schema_dir = 'nwb-schema/core'
 
-# copy requirements from requirements.txt, ignore all pinned version info, but keep pinned version for hdmf
-reqs_re = re.compile("[<=>]+")
-with open('requirements.txt', 'r') as fp:
-    reqs = [reqs_re.split(x.strip())[0] if not x.startswith('hdmf') else x.strip() for x in fp.readlines()]
+with open('requirements-min.txt', 'r') as fp:
+    # replace == with >= and remove trailing comments and spaces
+    reqs = [x.replace('==', '>=').split('#')[0].strip() for x in fp]
+    reqs = [x for x in reqs if x]  # remove empty strings
 
 print(reqs)
 

tox.ini

@@ -27,6 +27,14 @@ commands =
     coverage run test.py --pynwb
     coverage html -d tests/coverage/htmlcov
 
+# Test with python 3.5, pinned dev reqs, and minimum run requirements
+[testenv:py35-min-req]
+basepython = python3.5
+deps =
+    -rrequirements-dev.txt
+    -rrequirements-min.txt
+commands = {[testenv]commands}
+
 # Envs that builds wheels and source distribution
 [testenv:build]
 commands =
@@ -45,6 +53,13 @@ commands = {[testenv:build]commands}
 basepython = python3.7
 commands = {[testenv:build]commands}
 
+[testenv:build-py35-min-req]
+basepython = python3.5
+deps =
+    -rrequirements-dev.txt
+    -rrequirements-min.txt
+commands = {[testenv:build]commands}
+
 # Envs that will only be executed on CI that does coverage reporting
 [testenv:coverage]
 passenv = CODECOV_TOKEN
@@ -67,6 +82,10 @@ commands = python -c "import pynwb"
 deps = null
 commands = python -c "import pynwb"
 
+[testenv:wheelinstall-py35-min-req]
+deps = null
+commands = python -c "import pynwb"
+
 # Envs that will execute gallery tests
 [testenv:gallery]
 install_command =
@@ -89,3 +108,11 @@ commands = {[testenv:gallery]commands}
 basepython = python3.7
 deps = {[testenv:gallery]deps}
 commands = {[testenv:gallery]commands}
+
+[testenv:gallery-py36-min-req]
+basepython = python3.6
+deps =
+    -rrequirements-dev.txt
+    -rrequirements-min.txt
+    -rrequirements-doc.txt
+commands = {[testenv:gallery]commands}