From eb43890852f20fa4292b9aac0b184fb7a0642518 Mon Sep 17 00:00:00 2001 From: Umberto Lupo <46537483+ulupo@users.noreply.github.com> Date: Fri, 14 Feb 2020 13:01:22 +0100 Subject: [PATCH] Fix failures in CI ahead of first release, drop Python 3.5 (#11) * Silence junit_family warnings * Remove Python 3.5 support * Silence deprecation warnings for autodoc_default_flags * Download flag files from cloud in tests when testing from generic dirs * Bump Python version in README.rst * Extend cloud download approach to test_flagser, fix incorrect url * Correct dir path to file path * Linting * Use copyfile instead of copyfileobj * Use urlretrieve instead of urlopen * Add sudo permissions to pytest line in manylinux * Remove sudo, add conftest file defining --webdl, implement on flagio - Comment tests in test_flagser to silence failures * Try new approach to avoid issues with --pyargs and command line arguments * Add forgotten import dot and linting * Remove nightlies from Azure CI * Uninstall dev environment after testing it * Change flagser docstring formatting to avoid sphinx errors * Uninstall dev after testing dev in docker_scripts.sh Also replace dot in pytest with pyflagser * Avoid unnecessary coverage report generations and uploads * Wording changes in comments * Change bash to script in windows jobs * Remove contradictory passed options * Reintroduce coverage reports for all systems * Try with sudo in manylinux, pass /D in windows * Refactor tests * Update CODE_AUTHORS * UPDATE LICENSE * Update RELEASE * Update MAINTAINER variable in setup.py * Update GOVERNANCE * Better docstring formatting in flagser.py * Add sudo to pytest steps in linux systems * Remove sudo, create temp dir for pytests on wheels * Add sudo only in one place * Do not delete folders/files * Minor improvements in docs and variable names * Correct mistake in GOVERNANCE --- .azure-ci/docker_scripts.sh | 16 ++--- CODE_AUTHORS | 1 + GOVERNANCE.rst | 14 ++--- LICENSE | 2 +- README.rst | 2 +- RELEASE.rst | 2 +- azure-pipelines.yml | 105 +++++--------------------------- doc/conf.py | 2 +- pyflagser/_version.py | 4 +- pyflagser/flagio.py | 5 +- pyflagser/flagser.py | 55 ++++++++--------- pyflagser/tests/__main__.py | 9 +++ pyflagser/tests/conftest.py | 47 ++++++++++++++ pyflagser/tests/test_flagio.py | 15 +---- pyflagser/tests/test_flagser.py | 21 ++----- setup.cfg | 2 + setup.py | 8 +-- 17 files changed, 135 insertions(+), 175 deletions(-) create mode 100644 pyflagser/tests/__main__.py create mode 100644 pyflagser/tests/conftest.py diff --git a/.azure-ci/docker_scripts.sh b/.azure-ci/docker_scripts.sh index fd71d98..2895534 100644 --- a/.azure-ci/docker_scripts.sh +++ b/.azure-ci/docker_scripts.sh @@ -2,23 +2,25 @@ set -x -# upgrading pip and setuptools +# Upgrade pip and setuptools. TODO: Monitor status of pip versions PYTHON_PATH=$(eval find "/opt/python/*${python_ver}*" -print) export PATH=${PYTHON_PATH}/bin:${PATH} pip install --upgrade pip==19.3.1 setuptools -# installing cmake +# Install CMake pip install cmake -# installing and uninstalling pyflagser +# Install dev environment cd /io pip install -e ".[doc, tests]" -pip uninstall -y pyflagser -# testing, linting +# Test dev install with pytest and flake8 pytest --cov . --cov-report xml flake8 --exit-zero /io/ -# building wheels -pip install wheel twine +# Uninstal pyflagser dev +pip uninstall -y pyflagser + +# Build wheels +pip install wheel python setup.py sdist bdist_wheel diff --git a/CODE_AUTHORS b/CODE_AUTHORS index a4bf207..cab38a9 100644 --- a/CODE_AUTHORS +++ b/CODE_AUTHORS @@ -3,3 +3,4 @@ Guillaume Tauzin, guillaume.tauzin@epfl.ch Julian Burella Pérez, julian.burellaperez@heig-vd.ch +Umberto Lupo, u.lupo@l2f.ch diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst index ec3d993..dc6acd8 100644 --- a/GOVERNANCE.rst +++ b/GOVERNANCE.rst @@ -1,4 +1,4 @@ -This file describe the governance of the Giotto project. +This file describe the governance of the pyflagser project. Project owner: -------------- @@ -10,12 +10,10 @@ Authors: - Please refer to the `authors `_ file -Giotto Project Team: --------------------- +Pyflagser Project Team: +----------------------- -- Umberto Lupo u.lupo@l2f.ch (Maintainer & Developer) +- Guillaume Tauzin guillaume.tauzin@epfl.ch (Maintainer) +- Umberto Lupo u.lupo@l2f.ch (Maintainer) +- Julian Burella Pérez julian.burellaperez@heig-vd.ch (Developer) - Matteo Caorsi m.caorsi@l2f.ch (Project Leader) -- Philippe Nguyen p.nguyen@l2f.ch (Developer) - -Former Project Team Members: ----------------------------- diff --git a/LICENSE b/LICENSE index 37e986f..57c46d5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2019 L2F SA. +Copyright 2020 L2F SA. Licensed under the GNU Affero General Public License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License below or at https://www.gnu.org/licenses/agpl-3.0.html diff --git a/README.rst b/README.rst index 73c0d08..27f06ee 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ Dependencies pyflagser requires: -- Python (>= 3.5) +- Python (>= 3.6) - numpy (>= 1.17.0) - scipy (>= 0.17.0) diff --git a/RELEASE.rst b/RELEASE.rst index 9eed90f..6b7ed92 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -25,7 +25,7 @@ Thanks to our Contributors This release contains contributions from many people: -Guillaume Tauzin and Julian Burella Pérez. +Guillaume Tauzin, Julian Burella Pérez and Umberto Lupo. We are also grateful to all who filed issues or helped resolve them, asked and answered questions, and were part of inspiring discussions. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fabc2f9..6c95f1a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,8 +7,6 @@ jobs: vmImage: 'ubuntu-16.04' strategy: matrix: - Python35: - python.version: '3.5' Python36: python.version: '3.6' Python37: @@ -43,8 +41,6 @@ jobs: vmImage: 'macOS-10.14' strategy: matrix: - Python35: - python.version: '3.5' Python36: python.version: '3.6' Python37: @@ -85,9 +81,6 @@ jobs: vmImage: 'vs2017-win2016' strategy: matrix: - Python35: - python_ver: '35' - python.version: '3.5' Python36: python_ver: '36' python.version: '3.6' @@ -128,11 +121,6 @@ jobs: vmImage: 'ubuntu-16.04' strategy: matrix: - Python35: - arch: x86_64 - plat: manylinux2010_x86_64 - python_ver: '35' - python.version: '3.5' Python36: arch: x86_64 plat: manylinux2010_x86_64 @@ -154,14 +142,6 @@ jobs: inputs: versionSpec: '$(python.version)' - - bash: | - sed -i "s/'pyflagser'/'pyflagser-nightly'/1" setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" pyflagser/_version.py - cat pyflagser/_version.py - failOnStderr: true - condition: eq(variables['nightly_check'], 'true') - displayName: 'Change name to pyflagser-nightly' - - task: Bash@3 inputs: filePath: .azure-ci/build_manylinux2010.sh @@ -182,7 +162,7 @@ jobs: pip install pytest pytest-cov pytest-azurepipelines pytest-benchmark flake8 hypothesis mkdir tmp_test_cov cd tmp_test_cov - pytest --pyargs pyflagser --ignore-glob='flagser*' --no-cov --no-coverage-upload + python -m pyflagser.tests --webdl --no-cov --no-coverage-upload failOnStderr: true displayName: 'Test the wheels with pytest' @@ -198,13 +178,6 @@ jobs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'wheel_and_doc' - - bash: | - pip install twine - for f in dist/*linux* ; do sudo mv "$f" "${f/linux/manylinux2010}"; done - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/* - condition: eq(variables['nightly_check'], 'true') - displayName: 'Upload nightly wheels to PyPI' - - job: 'macOS1014' condition: eq(variables['build_check'], 'true') @@ -212,8 +185,6 @@ jobs: vmImage: 'macOS-10.14' strategy: matrix: - Python35: - python.version: '3.5' Python36: python.version: '3.6' Python37: @@ -226,16 +197,6 @@ jobs: inputs: versionSpec: '$(python.version)' - - bash: | - sed -i.bak "s/'pyflagser'/'pyflagser-nightly'/1" setup.py - rm setup.py.bak - sed -i.bak "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" pyflagser/_version.py - cat pyflagser/_version.py - rm pyflagser/_version.py.bak - failOnStderr: true - condition: eq(variables['nightly_check'], 'true') - displayName: 'Change name to pyflagser-nightly' - - script: | brew update brew install gcc @@ -251,24 +212,17 @@ jobs: failOnStderr: true displayName: 'Install dev environment' - - script: | - pip uninstall -y pyflagser - condition: eq(variables['nightly_check'], 'false') - failOnStderr: true - displayName: 'Uninstall pyflagser dev' - - - script: | - pip uninstall -y pyflagser-nightly - condition: eq(variables['nightly_check'], 'true') - failOnStderr: true - displayName: 'Uninstall pyflagser-nightly dev' - - script: | pytest --cov pyflagser --cov-report xml flake8 failOnStderr: true displayName: 'Test with pytest and flake8' + - script: | + pip uninstall -y pyflagser + failOnStderr: true + displayName: 'Uninstall pyflagser dev' + - script: | pip install wheel python setup.py sdist bdist_wheel @@ -282,7 +236,7 @@ jobs: - script: | mkdir tmp_test_cov cd tmp_test_cov - pytest --pyargs pyflagser --ignore-glob='flagser*' --no-cov --no-coverage-upload + python -m pyflagser.tests --webdl --no-cov --no-coverage-upload failOnStderr: true displayName: 'Test the wheels with pytest' @@ -314,12 +268,6 @@ jobs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'wheel_and_doc' - - bash: | - pip install twine - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/* - condition: eq(variables['nightly_check'], 'true') - displayName: 'Upload nightly wheels to PyPI' - - job: 'win2016' condition: eq(variables['build_check'], 'true') @@ -327,9 +275,6 @@ jobs: vmImage: 'vs2017-win2016' strategy: matrix: - Python35: - python_ver: '35' - python.version: '3.5' Python36: python_ver: '36' python.version: '3.6' @@ -345,14 +290,6 @@ jobs: inputs: versionSpec: '$(python.version)' - - bash: | - sed -i "s/'pyflagser'/'pyflagser-nightly'/1" setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" pyflagser/_version.py - cat pyflagser/_version.py - failOnStderr: true - condition: eq(variables['nightly_check'], 'true') - displayName: 'Change name to pyflagser-nightly' - - script: | python -m pip install --upgrade pip setuptools failOnStderr: true @@ -363,24 +300,17 @@ jobs: failOnStderr: true displayName: 'Install dev environment' - - script: | - pip uninstall -y pyflagser - condition: eq(variables['nightly_check'], 'false') - failOnStderr: true - displayName: 'Uninstall pyflagser dev' - - - script: | - pip uninstall -y pyflagser-nightly - condition: eq(variables['nightly_check'], 'true') - failOnStderr: true - displayName: 'Uninstall pyflagser-nightly dev' - - script: | pytest --cov pyflagser --cov-report xml flake8 failOnStderr: true displayName: 'Test with pytest and flake8' + - script: | + pip uninstall -y pyflagser + failOnStderr: true + displayName: 'Uninstall pyflagser dev' + - bash: | sed -i $'s/\r$//' README.rst pip install wheel @@ -392,10 +322,10 @@ jobs: failOnStderr: true displayName: 'Install the wheels' - - bash: | + - script: | mkdir tmp_test_cov cd tmp_test_cov - pytest --pyargs pyflagser --ignore-glob='flagser*' --no-cov --no-coverage-upload + python -m pyflagser.tests --webdl --no-cov --no-coverage-upload failOnStderr: true displayName: 'Test the wheels with pytest' @@ -410,10 +340,3 @@ jobs: inputs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'wheel_and_doc' - - - bash: | - pip install twine - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/* - failOnStderr: true - condition: eq(variables['nightly_check'], 'true') - displayName: 'Upload nightly wheels to PyPI' diff --git a/doc/conf.py b/doc/conf.py index c1e98af..4c610bf 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -58,7 +58,7 @@ mathjax_path = ('https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/' 'MathJax.js?config=TeX-AMS_SVG') -autodoc_default_flags = ['members', 'inherited-members'] +autodoc_default_options = {'members': True, 'inherited-members': True} # Add any paths that contain templates here, relative to this directory. templates_path = ['templates'] diff --git a/pyflagser/_version.py b/pyflagser/_version.py index a4830ca..76602af 100644 --- a/pyflagser/_version.py +++ b/pyflagser/_version.py @@ -1,6 +1,4 @@ -""" -``pyflagser`` is a python API for the C++ flagser library. -""" +"""``pyflagser`` is a python API for the C++ flagser library.""" # PEP0440 compatible formatted version, see: # https://www.python.org/dev/peps/pep-0440/ diff --git a/pyflagser/flagio.py b/pyflagser/flagio.py index dc60a61..028b9c6 100644 --- a/pyflagser/flagio.py +++ b/pyflagser/flagio.py @@ -5,12 +5,13 @@ def loadflag(fname, fmt='csr', dtype=None): - """Load a flag matrix from a ``.flag`` file. + """Load a ``.flag`` file, and return a matrix representation. Parameters ---------- fname : file, str, or pathlib.Path, required - Filename of extension ``.flag``. + Filename of extension ``.flag`` containing the information of a flag + matrix. fmt : {'dense', 'dia', 'csr', 'csc', 'lil', ...}, optional, default: 'csr' Matrix format of the result. By default, a CSR sparse matrix is diff --git a/pyflagser/flagser.py b/pyflagser/flagser.py index f522883..ea6290b 100644 --- a/pyflagser/flagser.py +++ b/pyflagser/flagser.py @@ -21,10 +21,10 @@ def flagser(flag_matrix, min_dimension=0, max_dimension=np.inf, directed=True, graph. Diagonal elements are vertex weights. min_dimension : int, optional, default: ``0`` - Minimal dimension. + Minimum homology dimension. - max_dimension : int, optional, default: ``np.inf`` - Maximum dimension. + max_dimension : int or np.inf, optional, default: ``np.inf`` + Maximum homology dimension. directed : bool, optional, default: ``True`` If true, computes the directed flag complex. Otherwise, it computes @@ -43,27 +43,26 @@ def flagser(flag_matrix, min_dimension=0, max_dimension=np.inf, directed=True, Returns ------- - out: dict of list + out : dict of list A dictionary holding the results of the flagser computation. Each value is a list of length `max_dimension` - `min_dimension`. The - structure of `out` is as follows: - { - 'dgms': list of ndarray of shape ``(n_pairs, 2)`` - A list of persistence diagrams, one for each dimension greater - than or equal than `min_dimension` and less than `max_dimension`. - Each diagram is an ndarray of size (n_pairs, 2) with the first - column representing the birth time and the second column - representing the death time of each pair. - 'cell_count': list of int - Cell count per dimension greater than or equal than - `min_dimension` and less than `max_dimension`. - 'betti': list of int - Betti number per dimension greater than or equal than - `min_dimension` and less than `max_dimension`. - 'euler': list of int - Euler characteristic per dimension greater than or equal than - `min_dimension` and less than `max_dimension`. - } + key-value pairs in `out` are as follows: + + - ``'dgms'``: list of ndarray of shape ``(n_pairs, 2)`` + A list of persistence diagrams, one for each dimension greater + than or equal than `min_dimension` and less than `max_dimension`. + Each diagram is an ndarray of size (n_pairs, 2) with the first + column representing the birth time and the second column + representing the death time of each pair. + - ``'cell_count'``: list of int + Cell count per dimension greater than or equal than + `min_dimension` and less than `max_dimension`. + - ``'betti'``: list of int + Betti number per dimension greater than or equal than + `min_dimension` and less than `max_dimension`. + - ``'euler'``: list of int + Euler characteristic per dimension greater than or equal than + `min_dimension` and less than `max_dimension`. """ vertices = np.asarray(flag_matrix.diagonal()).copy() @@ -93,10 +92,10 @@ def flagser(flag_matrix, min_dimension=0, max_dimension=np.inf, directed=True, homology = compute_homology(vertices, edges, min_dimension, _max_dimension, directed, coeff, approximation) # Creating dictionary of return values - ret = dict() - ret['dgms'] = homology[0].get_persistence_diagram() - ret['cell_count'] = homology[0].get_cell_count() - ret['betti'] = homology[0].get_betti_numbers() - ret['euler'] = homology[0].get_euler_characteristic() + out = dict() + out['dgms'] = homology[0].get_persistence_diagram() + out['cell_count'] = homology[0].get_cell_count() + out['betti'] = homology[0].get_betti_numbers() + out['euler'] = homology[0].get_euler_characteristic() - return ret + return out diff --git a/pyflagser/tests/__main__.py b/pyflagser/tests/__main__.py new file mode 100644 index 0000000..03a3e61 --- /dev/null +++ b/pyflagser/tests/__main__.py @@ -0,0 +1,9 @@ +import os +import sys + +HERE = os.path.dirname(__file__) + +if __name__ == "__main__": + import pytest + errcode = pytest.main([HERE] + sys.argv[1:]) + sys.exit(errcode) diff --git a/pyflagser/tests/conftest.py b/pyflagser/tests/conftest.py new file mode 100644 index 0000000..834484b --- /dev/null +++ b/pyflagser/tests/conftest.py @@ -0,0 +1,47 @@ +import os +from tempfile import mkdtemp +from urllib.request import urlopen, urlretrieve + + +def pytest_addoption(parser): + parser.addoption('--webdl', action='store_true', default=False, + help='Whether or not to download files ' + 'required for testing from the web') + + +def fetch_flag_files(webdl): + if not webdl: + dirname = os.path.join(os.path.dirname(__file__), '../../flagser/test') + try: + fnames = os.listdir(dirname) + flag_files = [os.path.join(dirname, fname) for fname in fnames + if fname.endswith(".flag")] + except FileNotFoundError: + print(f'.flag files looked for in {dirname}, but directory does ' + 'not exist. Pass the optional argument --webdl to ' + 'automatically download these files into a temporary ' + 'folder and run tests.') + else: + # Download from remote bucket + temp_dir = mkdtemp() + bucket_url = 'https://storage.googleapis.com/l2f-open-models/' \ + 'giotto-tda/flagser/test/' + flag_files_list = bucket_url + 'flag_files_list.txt' + with urlopen(flag_files_list) as f: + flag_file_names = f.read().decode('utf8').splitlines() + flag_files = [] + for fname in flag_file_names: + url = bucket_url + fname + fpath = os.path.join(temp_dir, fname) + urlretrieve(url, fpath) + flag_files.append(fpath) + return flag_files + + +def pytest_generate_tests(metafunc): + # This is called for every test. Only get/set command line arguments + # if the argument is specified in the list of test "fixturenames". + webdl = metafunc.config.option.webdl + if 'flag_file' in metafunc.fixturenames: + flag_files = fetch_flag_files(webdl) + metafunc.parametrize('flag_file', flag_files) diff --git a/pyflagser/tests/test_flagio.py b/pyflagser/tests/test_flagio.py index 468417d..0ef1573 100644 --- a/pyflagser/tests/test_flagio.py +++ b/pyflagser/tests/test_flagio.py @@ -1,26 +1,17 @@ """Testing for the python bindings of the C++ flagser library.""" +import os + import numpy as np import scipy.sparse as sp -import pytest -import os from numpy.testing import assert_almost_equal from pyflagser import loadflag, saveflag -flag_files = [] - -dirname = os.path.join(os.path.dirname(__file__), "../../flagser/test") -for file in os.listdir(dirname): - if file.endswith(".flag"): - flag_files.append(os.path.join(dirname, file)) - -@pytest.mark.parametrize("flag_file", - [(flag_file) for flag_file in flag_files]) def test_flagio(flag_file): flag_matrix = loadflag(flag_file) - _, fname_temp = os.path.split(flag_file) + fname_temp = os.path.split(flag_file)[1] saveflag(fname_temp, flag_matrix) flag_matrix_temp = loadflag(fname_temp) os.remove(fname_temp) diff --git a/pyflagser/tests/test_flagser.py b/pyflagser/tests/test_flagser.py index e632a00..75f2497 100644 --- a/pyflagser/tests/test_flagser.py +++ b/pyflagser/tests/test_flagser.py @@ -1,18 +1,11 @@ """Testing for the python bindings of the C++ flagser library.""" -import pytest import os + from numpy.testing import assert_almost_equal from pyflagser import loadflag, flagser -flag_files = [] - -dirname = os.path.join(os.path.dirname(__file__), "../../flagser/test") -for file in os.listdir(dirname): - if file.endswith(".flag"): - flag_files.append(os.path.join(dirname, file)) - betti = { 'a.flag': [1, 2, 0], 'b.flag': [1, 0, 0], @@ -34,12 +27,8 @@ } -@pytest.mark.parametrize("flag_file, betti", - [(flag_file, betti[os.path.split(flag_file)[1]]) - for flag_file in flag_files - if os.path.split(flag_file)[1] in betti.keys()]) -def test_flagser(flag_file, betti): +def test_flagser(flag_file): + betti_exp = betti[os.path.split(flag_file)[1]] flag_matrix = loadflag(flag_file) - - ret = flagser(flag_matrix) - assert_almost_equal(ret['betti'], betti) + betti_res = flagser(flag_matrix)['betti'] + assert_almost_equal(betti_res, betti_exp) diff --git a/setup.cfg b/setup.cfg index 363b597..b03d0a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,8 @@ description-file = README.rst [tool:pytest] +junit_family=xunit1 + addopts = --ignore flagser --ignore pybind11 diff --git a/setup.py b/setup.py index 44401c9..909ab33 100755 --- a/setup.py +++ b/setup.py @@ -24,10 +24,10 @@ with codecs.open('README.rst', encoding='utf-8-sig') as f: LONG_DESCRIPTION = f.read() LONG_DESCRIPTION_TYPE = 'text/x-rst' -MAINTAINER = 'Guillaume Tauzin' +MAINTAINER = 'Guillaume Tauzin, Umberto Lupo' MAINTAINER_EMAIL = 'maintainers@giotto.ai' URL = 'https://github.com/giotto-ai/pyflagser' -LICENSE = 'AGPLv3' +LICENSE = 'GNU AGPLv3' DOWNLOAD_URL = 'https://github.com/giotto-ai/pyflagser/tarball/v0.1.0' VERSION = __version__ # noqa CLASSIFIERS = ['Intended Audience :: Science/Research', @@ -41,9 +41,9 @@ 'Operating System :: POSIX', 'Operating System :: Unix', 'Operating System :: MacOS', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7'] + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8'] KEYWORDS = 'topological data analysis, persistent ' + \ 'homology, directed flags complex, persistence diagrams' INSTALL_REQUIRES = requirements