From 6bb1d771c014760546d4037340c682f294357ccd Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 16:47:26 +0100 Subject: [PATCH 1/8] Support Python 3.12 --- .github/workflows/test.yml | 10 +++++++++- pyproject.toml | 5 +++-- tests/test_api.py | 2 +- tests/test_backend_filesystem.py | 21 +++++++++++++++++++++ tests/test_backend_minio.py | 23 ++++++++++++++++++++++- tests/test_interface_unversioned.py | 5 ++++- tests/test_legacy_import.py | 3 ++- 7 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf4a3b07..44ea4593 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,8 @@ jobs: python-version: '3.9' - os: ubuntu-latest python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' steps: - uses: actions/checkout@v4 @@ -48,12 +50,18 @@ jobs: run: | pip install -r requirements.txt - - name: Test with pytest + - name: Test with pytest for Python <3.12 env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }} run: | python -m pytest --cov-config=.coveragerc.${{ runner.os }} + if: matrix.python-version != '3.12' + + - name: Test with pytest for Python 3.12 + run: | + python -m pytest --cov-config=.coveragerc.python3.12 --ignore=audbackend/core/backend/artifactory.py --ignore=tests/test_backend_artifactory.py + if: matrix.python-version == '3.12' - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/pyproject.toml b/pyproject.toml index fd0feea3..f040e62e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Scientific/Engineering', ] dependencies = [ @@ -37,13 +38,13 @@ dynamic = ['version'] [project.optional-dependencies] artifactory = [ - 'dohq-artifactory >=0.10.0', + 'dohq-artifactory >=0.10.0; python_version < "3.12"', ] minio = [ 'minio', ] all = [ - 'dohq-artifactory >=0.10.0', + 'dohq-artifactory >=0.10.0; python_version < "3.12"', 'minio', ] diff --git a/tests/test_api.py b/tests/test_api.py index 584c3d3d..4a6f5027 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -18,7 +18,7 @@ "artifactory", "artifactory", f"unittest-{audeer.uid()[:8]}", - audbackend.backend.Artifactory, + pytest.importorskip("audbackend.backend.Artifactory"), ), ( "minio", diff --git a/tests/test_backend_filesystem.py b/tests/test_backend_filesystem.py index 1ed2a9ac..fc0d3a43 100644 --- a/tests/test_backend_filesystem.py +++ b/tests/test_backend_filesystem.py @@ -167,3 +167,24 @@ def test_maven_file_structure( assert path_expected == path assert interface.ls(file) == [(file, version)] assert interface.ls() == [(file, version)] + + +def test_errors(tmpdir): + r"""Test error for creating and opening backend. + + Args: + tmpdir: tmpdir fixture + + """ + host = audeer.mkdir(tmpdir, "backend") + repo = "repo" + backend = audbackend.backend.FileSystem(host, repo) + with pytest.raises(FileNotFoundError): + backend._open() + backend._create() + assert os.path.exists(os.path.join(host, repo)) + backend._open() + backend._close() + with pytest.raises(FileExistsError): + backend._create() + backend._delete() diff --git a/tests/test_backend_minio.py b/tests/test_backend_minio.py index 6fc71e61..4cfa620f 100644 --- a/tests/test_backend_minio.py +++ b/tests/test_backend_minio.py @@ -144,12 +144,33 @@ def test_create_delete_repositories(host, repository): @pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) @pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) @pytest.mark.parametrize("authentication", [("bad-access", "bad-secret")]) -def test_errors(host, repository, authentication): +def test_errors_authentication(host, repository, authentication): backend = audbackend.backend.Minio(host, repository, authentication=authentication) with pytest.raises(audbackend.BackendError): backend.open() +@pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) +@pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) +def test_errors(host, repository): + r"""Test error for creating and opening backend. + + Args: + host: host + repository: repository + + """ + backend = audbackend.backend.Minio(host, repository) + with pytest.raises(FileNotFoundError): + backend._open() + backend._create() + backend._open() + backend._close() + with pytest.raises(FileExistsError): + backend._create() + backend._delete() + + def test_get_config(tmpdir, hosts, hide_credentials): r"""Test parsing of configuration. diff --git a/tests/test_interface_unversioned.py b/tests/test_interface_unversioned.py index b5cf432c..ec153a1d 100644 --- a/tests/test_interface_unversioned.py +++ b/tests/test_interface_unversioned.py @@ -17,11 +17,14 @@ # Backend-interface combinations to use in all tests backend_interface_combinations = [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), (audbackend.backend.FileSystem, audbackend.interface.Unversioned), (audbackend.backend.Minio, audbackend.interface.Unversioned), (SingleFolder, audbackend.interface.Unversioned), ] +if hasattr(audbackend.backend, "Artifactory"): + backend_interface_combinations.append( + (audbackend.backend.Artifactory, audbackend.interface.Unversioned) + ) @pytest.fixture(scope="function", autouse=False) diff --git a/tests/test_legacy_import.py b/tests/test_legacy_import.py index 3a65e3da..b67877bc 100644 --- a/tests/test_legacy_import.py +++ b/tests/test_legacy_import.py @@ -3,5 +3,6 @@ def test_legacy_import(hosts): audbackend.Backend("host", "repo") - audbackend.Artifactory(hosts["artifactory"], "repo") audbackend.FileSystem(hosts["file-system"], "repo") + if hasattr(audbackend, "Artifactory"): + audbackend.Artifactory(hosts["artifactory"], "repo") From 2ac06f0a236e4c515c153409778ad7d796e34c7e Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 16:53:18 +0100 Subject: [PATCH 2/8] Add coverage file for Python 3.12 --- .coveragerc.python3.12 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .coveragerc.python3.12 diff --git a/.coveragerc.python3.12 b/.coveragerc.python3.12 new file mode 100644 index 00000000..7e23663d --- /dev/null +++ b/.coveragerc.python3.12 @@ -0,0 +1,7 @@ +[report] +exclude_lines = + pragma: no cover + pragma: no Linux cover +omit = + audbackend/core/api.py + audbackend/core/backend/artifactory.py From da9c6537d0cea3f17483c33cb6f3c2423781a178 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 16:56:34 +0100 Subject: [PATCH 3/8] Document the absence of Artifactory in 3.12 --- README.rst | 4 ++-- docs/install.rst | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2cbf31b6..11dd879b 100644 --- a/README.rst +++ b/README.rst @@ -18,11 +18,11 @@ the requested data structure in a repository on a storage system, such as a file system -or Artifactory_. +or MinIO_. Have a look at the installation_ and usage_ instructions. -.. _Artifactory: https://jfrog.com/artifactory/ +.. _MinIO: https://min.io .. _backends: https://audeering.github.io/audbackend/api/audbackend.backend.html .. _interfaces: https://audeering.github.io/audbackend/api/audbackend.interface.html .. _installation: https://audeering.github.io/audbackend/install.html diff --git a/docs/install.rst b/docs/install.rst index d0911b32..84a1e201 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -25,3 +25,7 @@ e.g. :class:`audbackend.backend.Artifactory` .. code-block:: bash $ pip install audbackend[artifactory] + +Note, +in Python 3.12 the :class:`audbackend.backend.Artifactory` +backend is no longer available. From 4ca08bc0eec56b24523a2c38019740600a6e51e7 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 19:05:12 +0100 Subject: [PATCH 4/8] Don't upload cov for Python 3.12 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44ea4593..a0f3a781 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,4 +68,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-latest' and matrix.python-version != '3.12' From 515695e7538c945a03200581d411cb4e4efdc682 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 19:36:02 +0100 Subject: [PATCH 5/8] Update api testing --- tests/test_api.py | 22 ++++++++++++++-------- tests/test_backend_filesystem.py | 21 --------------------- tests/test_backend_minio.py | 23 +---------------------- 3 files changed, 15 insertions(+), 51 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 4a6f5027..c6caa808 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,5 @@ +import sys + import pytest import audeer @@ -6,25 +8,29 @@ @pytest.mark.parametrize( - "name, host, repository, cls", + "name, host, repository, expected_class", [ ( "file-system", "file-system", f"unittest-{audeer.uid()[:8]}", - audbackend.backend.FileSystem, + "audbackend.backend.FileSystem", ), - ( + pytest.param( "artifactory", "artifactory", f"unittest-{audeer.uid()[:8]}", - pytest.importorskip("audbackend.backend.Artifactory"), + "audbackend.backend.Artifactory", + marks=pytest.mark.skipif( + sys.version_info > (3, 11), + reason="Requires Python 3.11 or lower", + ), ), ( "minio", "minio", f"unittest-{audeer.uid()[:8]}", - audbackend.backend.Minio, + "audbackend.backend.Minio", ), pytest.param( # backend does not exist "bad-backend", @@ -49,7 +55,7 @@ ), ], ) -def test_api(hosts, name, host, repository, cls): +def test_api(hosts, name, host, repository, expected_class): if host is not None and host in hosts: host = hosts[name] @@ -84,7 +90,7 @@ def test_api(hosts, name, host, repository, cls): with pytest.warns(UserWarning, match=create_warning): interface = audbackend.create(name, host, repository) assert isinstance(interface, audbackend.interface.Versioned) - assert isinstance(interface.backend, cls) + assert str(interface.backend).startswith(expected_class) with pytest.raises(audbackend.BackendError, match=error_msg): with pytest.warns(UserWarning, match=create_warning): @@ -92,7 +98,7 @@ def test_api(hosts, name, host, repository, cls): with pytest.warns(UserWarning, match=access_warning): interface = audbackend.access(name, host, repository) - assert isinstance(interface.backend, cls) + assert str(interface.backend).startswith(expected_class) with pytest.warns(UserWarning, match=delete_warning): audbackend.delete(name, host, repository) diff --git a/tests/test_backend_filesystem.py b/tests/test_backend_filesystem.py index fc0d3a43..1ed2a9ac 100644 --- a/tests/test_backend_filesystem.py +++ b/tests/test_backend_filesystem.py @@ -167,24 +167,3 @@ def test_maven_file_structure( assert path_expected == path assert interface.ls(file) == [(file, version)] assert interface.ls() == [(file, version)] - - -def test_errors(tmpdir): - r"""Test error for creating and opening backend. - - Args: - tmpdir: tmpdir fixture - - """ - host = audeer.mkdir(tmpdir, "backend") - repo = "repo" - backend = audbackend.backend.FileSystem(host, repo) - with pytest.raises(FileNotFoundError): - backend._open() - backend._create() - assert os.path.exists(os.path.join(host, repo)) - backend._open() - backend._close() - with pytest.raises(FileExistsError): - backend._create() - backend._delete() diff --git a/tests/test_backend_minio.py b/tests/test_backend_minio.py index 4cfa620f..6fc71e61 100644 --- a/tests/test_backend_minio.py +++ b/tests/test_backend_minio.py @@ -144,33 +144,12 @@ def test_create_delete_repositories(host, repository): @pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) @pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) @pytest.mark.parametrize("authentication", [("bad-access", "bad-secret")]) -def test_errors_authentication(host, repository, authentication): +def test_errors(host, repository, authentication): backend = audbackend.backend.Minio(host, repository, authentication=authentication) with pytest.raises(audbackend.BackendError): backend.open() -@pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) -@pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) -def test_errors(host, repository): - r"""Test error for creating and opening backend. - - Args: - host: host - repository: repository - - """ - backend = audbackend.backend.Minio(host, repository) - with pytest.raises(FileNotFoundError): - backend._open() - backend._create() - backend._open() - backend._close() - with pytest.raises(FileExistsError): - backend._create() - backend._delete() - - def test_get_config(tmpdir, hosts, hide_credentials): r"""Test parsing of configuration. From 87f8e89ab5d7a0db231bc65e0f83191bfb581224 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 19:46:16 +0100 Subject: [PATCH 6/8] Fix test for Python 3.12 --- tests/test_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index c6caa808..87773a3f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -40,15 +40,15 @@ marks=pytest.mark.xfail(raises=ValueError), ), pytest.param( # host does not exist - "artifactory", + "minio", "bad-host", "repo", None, marks=pytest.mark.xfail(raises=audbackend.BackendError), ), pytest.param( # invalid repository name - "artifactory", - "artifactory", + "minio", + "minio", "bad/repo", None, marks=pytest.mark.xfail(raises=audbackend.BackendError), From e2e26c86dfb952d93bc57b08b1ab9e0f0dd7657a Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 19:52:36 +0100 Subject: [PATCH 7/8] Update docs --- docs/install.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 84a1e201..22dec054 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -20,7 +20,13 @@ To install all backends run: $ pip install audbackend[all] or select single backends, -e.g. :class:`audbackend.backend.Artifactory` +e.g. :class:`audbackend.backend.Minio` + +.. code-block:: bash + + $ pip install audbackend[minio] + +or :class:`audbackend.backend.Artifactory` .. code-block:: bash @@ -28,4 +34,4 @@ e.g. :class:`audbackend.backend.Artifactory` Note, in Python 3.12 the :class:`audbackend.backend.Artifactory` -backend is no longer available. +backend is not available at the moment. From adcca5d1782ec299d54b07f9875186601b3a8db3 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 30 Oct 2024 19:59:34 +0100 Subject: [PATCH 8/8] Update docs --- docs/install.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 22dec054..eeb5e115 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,14 +19,14 @@ To install all backends run: $ pip install audbackend[all] -or select single backends, -e.g. :class:`audbackend.backend.Minio` +You can also select single backends, +e.g. :class:`audbackend.backend.Minio`: .. code-block:: bash $ pip install audbackend[minio] -or :class:`audbackend.backend.Artifactory` +or :class:`audbackend.backend.Artifactory`: .. code-block:: bash