From 8e3faf847b621a9209079bdd798159ff14af449b Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Tue, 9 Nov 2021 09:12:31 +0100 Subject: [PATCH 1/8] FIX: correct module name retrieval in `backend.plugins.remove_duplicates()` --- xarray/backends/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index bcaee498b90..257594925f8 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -26,8 +26,8 @@ def remove_duplicates(entrypoints): unique_entrypoints.append(matches[0]) matches_len = len(matches) if matches_len > 1: - selected_module_name = matches[0].module_name - all_module_names = [e.module_name for e in matches] + selected_module_name = matches[0].name + all_module_names = [e.name for e in matches] warnings.warn( f"Found {matches_len} entrypoints for the engine name {name}:" f"\n {all_module_names}.\n It will be used: {selected_module_name}.", From d64995327caeee02b86fb5ea2cde634c90aeac8b Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Tue, 9 Nov 2021 13:15:45 +0100 Subject: [PATCH 2/8] remove equal entrypoints, use `entrypoint.value` to extract module name --- xarray/backends/plugins.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 257594925f8..9ab0e264319 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -22,15 +22,17 @@ def remove_duplicates(entrypoints): # check if there are multiple entrypoints for the same name unique_entrypoints = [] for name, matches in entrypoints_grouped: - matches = list(matches) + # remove equal entrypoints + matches = list(set(matches)) unique_entrypoints.append(matches[0]) matches_len = len(matches) if matches_len > 1: - selected_module_name = matches[0].name - all_module_names = [e.name for e in matches] + all_module_names = [e.value.split(":")[0] for e in matches] + selected_module_name = all_module_names[0] warnings.warn( f"Found {matches_len} entrypoints for the engine name {name}:" - f"\n {all_module_names}.\n It will be used: {selected_module_name}.", + f"\n {all_module_names}.\n " + f"The entrypoint {selected_module_name} will be used.", RuntimeWarning, ) return unique_entrypoints From 4cfe4bab9d464532193ca6a262f29ecf0a2021de Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Tue, 9 Nov 2021 14:04:02 +0100 Subject: [PATCH 3/8] FIX: switch from pkg_resources to importlib in test_plugins.py --- xarray/tests/test_plugins.py | 48 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 7f77a677d6d..6263a0b580f 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -1,6 +1,13 @@ from unittest import mock -import pkg_resources +try: + import importlib.metadata as importlib_metadata +except ImportError: + # if the fallback library is missing, we are doomed. + import importlib_metadata as importlib_metadata + +from importlib_metadata import EntryPoint + import pytest from xarray.backends import common, plugins @@ -29,25 +36,26 @@ def open_dataset(self, filename_or_obj, *, decoder): @pytest.fixture def dummy_duplicated_entrypoints(): specs = [ - "engine1 = xarray.tests.test_plugins:backend_1", - "engine1 = xarray.tests.test_plugins:backend_2", - "engine2 = xarray.tests.test_plugins:backend_1", - "engine2 = xarray.tests.test_plugins:backend_2", + ["engine1", "xarray.tests.test_plugins:backend_1", "xarray.backends"], + ["engine1", "xarray.tests.test_plugins:backend_2", "xarray.backends"], + ["engine2", "xarray.tests.test_plugins:backend_1", "xarray.backends"], + ["engine2", "xarray.tests.test_plugins:backend_2", "xarray.backends"], ] - eps = [pkg_resources.EntryPoint.parse(spec) for spec in specs] + eps = [EntryPoint(name, value, group) for name, value, group in specs] return eps @pytest.mark.filterwarnings("ignore:Found") def test_remove_duplicates(dummy_duplicated_entrypoints) -> None: + entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints) with pytest.warns(RuntimeWarning): entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints) assert len(entrypoints) == 2 def test_broken_plugin() -> None: - broken_backend = pkg_resources.EntryPoint.parse( - "broken_backend = xarray.tests.test_plugins:backend_1" + broken_backend = EntryPoint( + "broken_backend", "xarray.tests.test_plugins:backend_1", "xarray.backends", ) with pytest.warns(RuntimeWarning) as record: _ = plugins.build_engines([broken_backend]) @@ -68,13 +76,13 @@ def test_remove_duplicates_warnings(dummy_duplicated_entrypoints) -> None: assert "entrypoints" in message1 -@mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=None)) +@mock.patch("importlib_metadata.EntryPoint.load", mock.MagicMock(return_value=None)) def test_backends_dict_from_pkg() -> None: specs = [ - "engine1 = xarray.tests.test_plugins:backend_1", - "engine2 = xarray.tests.test_plugins:backend_2", + ["engine1", "xarray.tests.test_plugins:backend_1", "xarray.backends"], + ["engine2", "xarray.tests.test_plugins:backend_2", "xarray.backends"] ] - entrypoints = [pkg_resources.EntryPoint.parse(spec) for spec in specs] + entrypoints = [EntryPoint(name, value, group) for name, value, group in specs] engines = plugins.backends_dict_from_pkg(entrypoints) assert len(engines) == 2 assert engines.keys() == set(("engine1", "engine2")) @@ -114,12 +122,12 @@ def test_set_missing_parameters_raise_error() -> None: @mock.patch( - "pkg_resources.EntryPoint.load", + "importlib_metadata.EntryPoint.load", mock.MagicMock(return_value=DummyBackendEntrypoint1), ) def test_build_engines() -> None: - dummy_pkg_entrypoint = pkg_resources.EntryPoint.parse( - "cfgrib = xarray.tests.test_plugins:backend_1" + dummy_pkg_entrypoint = EntryPoint( + "cfgrib", "xarray.tests.test_plugins:backend_1", "xarray_backends" ) backend_entrypoints = plugins.build_engines([dummy_pkg_entrypoint]) @@ -131,16 +139,16 @@ def test_build_engines() -> None: @mock.patch( - "pkg_resources.EntryPoint.load", + "importlib_metadata.EntryPoint.load", mock.MagicMock(return_value=DummyBackendEntrypoint1), ) def test_build_engines_sorted() -> None: dummy_pkg_entrypoints = [ - pkg_resources.EntryPoint.parse( - "dummy2 = xarray.tests.test_plugins:backend_1", + EntryPoint( + "dummy2", "xarray.tests.test_plugins:backend_1", "xarray.backends" ), - pkg_resources.EntryPoint.parse( - "dummy1 = xarray.tests.test_plugins:backend_1", + EntryPoint( + "dummy1", "xarray.tests.test_plugins:backend_1", "xarray.backends" ), ] backend_entrypoints = plugins.build_engines(dummy_pkg_entrypoints) From bd6db7947e762a53a5d55e3a8908dc31c2e9e6aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Nov 2021 13:06:32 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_plugins.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 6263a0b580f..99d10dc9de2 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -1,17 +1,16 @@ from unittest import mock +import pytest +from importlib_metadata import EntryPoint + +from xarray.backends import common, plugins + try: import importlib.metadata as importlib_metadata except ImportError: # if the fallback library is missing, we are doomed. import importlib_metadata as importlib_metadata -from importlib_metadata import EntryPoint - -import pytest - -from xarray.backends import common, plugins - class DummyBackendEntrypointArgs(common.BackendEntrypoint): def open_dataset(filename_or_obj, *args): @@ -55,7 +54,9 @@ def test_remove_duplicates(dummy_duplicated_entrypoints) -> None: def test_broken_plugin() -> None: broken_backend = EntryPoint( - "broken_backend", "xarray.tests.test_plugins:backend_1", "xarray.backends", + "broken_backend", + "xarray.tests.test_plugins:backend_1", + "xarray.backends", ) with pytest.warns(RuntimeWarning) as record: _ = plugins.build_engines([broken_backend]) @@ -80,7 +81,7 @@ def test_remove_duplicates_warnings(dummy_duplicated_entrypoints) -> None: def test_backends_dict_from_pkg() -> None: specs = [ ["engine1", "xarray.tests.test_plugins:backend_1", "xarray.backends"], - ["engine2", "xarray.tests.test_plugins:backend_2", "xarray.backends"] + ["engine2", "xarray.tests.test_plugins:backend_2", "xarray.backends"], ] entrypoints = [EntryPoint(name, value, group) for name, value, group in specs] engines = plugins.backends_dict_from_pkg(entrypoints) @@ -144,12 +145,8 @@ def test_build_engines() -> None: ) def test_build_engines_sorted() -> None: dummy_pkg_entrypoints = [ - EntryPoint( - "dummy2", "xarray.tests.test_plugins:backend_1", "xarray.backends" - ), - EntryPoint( - "dummy1", "xarray.tests.test_plugins:backend_1", "xarray.backends" - ), + EntryPoint("dummy2", "xarray.tests.test_plugins:backend_1", "xarray.backends"), + EntryPoint("dummy1", "xarray.tests.test_plugins:backend_1", "xarray.backends"), ] backend_entrypoints = plugins.build_engines(dummy_pkg_entrypoints) backend_entrypoints = list(backend_entrypoints) From ee89d9245f45b246e6313d64308cd5994403fbad Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Tue, 9 Nov 2021 14:17:06 +0100 Subject: [PATCH 5/8] Fix imports --- xarray/tests/test_plugins.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 99d10dc9de2..6fe865a91e4 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -1,15 +1,18 @@ from unittest import mock import pytest -from importlib_metadata import EntryPoint from xarray.backends import common, plugins try: import importlib.metadata as importlib_metadata + + from importlib_metadata import EntryPoint + except ImportError: # if the fallback library is missing, we are doomed. - import importlib_metadata as importlib_metadata + import importlib_metadata as importlib_metadata # type: ignore[no-redef] + from importlib_metadata import EntryPoint # type: ignore[no-redef] class DummyBackendEntrypointArgs(common.BackendEntrypoint): From 823778d9f6e378884d0f24e07660015f98ac03fd Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Tue, 9 Nov 2021 14:58:51 +0100 Subject: [PATCH 6/8] WIP: remove stale code, add whats-new.rst entry --- doc/whats-new.rst | 6 ++++++ xarray/tests/test_plugins.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index f499cbe3d21..b66c99d0bcb 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,8 @@ Bug fixes ~~~~~~~~~ - Fix plot.line crash for data of shape ``(1, N)`` in _title_for_slice on format_item (:pull:`5948`). By `Sebastian Weigand `_. +- Fix a regression in the removal of duplicate backend entrypoints (:issue:`5944`, :pull:`5959`) + By `Kai Mühlbauer `_. Documentation ~~~~~~~~~~~~~ @@ -49,6 +51,10 @@ Documentation Internal Changes ~~~~~~~~~~~~~~~~ +- Use ``importlib`` to replace functionality of ``pkg_resources`` in + backend plugins tests. (:pull:`5959`). + By `Kai Mühlbauer `_. + .. _whats-new.0.20.1: diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 6fe865a91e4..086beba97f8 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -49,7 +49,6 @@ def dummy_duplicated_entrypoints(): @pytest.mark.filterwarnings("ignore:Found") def test_remove_duplicates(dummy_duplicated_entrypoints) -> None: - entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints) with pytest.warns(RuntimeWarning): entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints) assert len(entrypoints) == 2 From e132893bef017c15e37a9d11d9fc6d14bf0e4ea2 Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Wed, 10 Nov 2021 08:39:08 +0100 Subject: [PATCH 7/8] WIP: change imports per review discussion --- xarray/tests/test_plugins.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 086beba97f8..2844e8b8c25 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -5,15 +5,15 @@ from xarray.backends import common, plugins try: - import importlib.metadata as importlib_metadata - - from importlib_metadata import EntryPoint + from importlib.metadata import EntryPoint + importlib_metadata_mock = "importlib.metadata" except ImportError: # if the fallback library is missing, we are doomed. - import importlib_metadata as importlib_metadata # type: ignore[no-redef] from importlib_metadata import EntryPoint # type: ignore[no-redef] + importlib_metadata_mock = "importlib_metadata" + class DummyBackendEntrypointArgs(common.BackendEntrypoint): def open_dataset(filename_or_obj, *args): @@ -79,7 +79,9 @@ def test_remove_duplicates_warnings(dummy_duplicated_entrypoints) -> None: assert "entrypoints" in message1 -@mock.patch("importlib_metadata.EntryPoint.load", mock.MagicMock(return_value=None)) +@mock.patch( + f"{importlib_metadata_mock}.EntryPoint.load", mock.MagicMock(return_value=None) +) def test_backends_dict_from_pkg() -> None: specs = [ ["engine1", "xarray.tests.test_plugins:backend_1", "xarray.backends"], @@ -125,7 +127,7 @@ def test_set_missing_parameters_raise_error() -> None: @mock.patch( - "importlib_metadata.EntryPoint.load", + f"{importlib_metadata_mock}.EntryPoint.load", mock.MagicMock(return_value=DummyBackendEntrypoint1), ) def test_build_engines() -> None: @@ -142,7 +144,7 @@ def test_build_engines() -> None: @mock.patch( - "importlib_metadata.EntryPoint.load", + f"{importlib_metadata_mock}.EntryPoint.load", mock.MagicMock(return_value=DummyBackendEntrypoint1), ) def test_build_engines_sorted() -> None: From 697b112b179b4056c532e86c1de4377c7fb25131 Mon Sep 17 00:00:00 2001 From: Kai Muehlbauer Date: Sun, 14 Nov 2021 16:29:39 +0100 Subject: [PATCH 8/8] fix mypy issue with importlib imports, see #5979 --- xarray/backends/plugins.py | 7 ++++--- xarray/tests/test_plugins.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 9ab0e264319..0a9ffcbda22 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -1,15 +1,16 @@ import functools import inspect import itertools +import sys import warnings from .common import BACKEND_ENTRYPOINTS, BackendEntrypoint -try: +if sys.version_info >= (3, 8): from importlib.metadata import entry_points -except ImportError: +else: # if the fallback library is missing, we are doomed. - from importlib_metadata import entry_points # type: ignore[no-redef] + from importlib_metadata import entry_points STANDARD_BACKENDS_ORDER = ["netcdf4", "h5netcdf", "scipy"] diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 2844e8b8c25..4d1eee6363d 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -1,16 +1,17 @@ +import sys from unittest import mock import pytest from xarray.backends import common, plugins -try: +if sys.version_info >= (3, 8): from importlib.metadata import EntryPoint importlib_metadata_mock = "importlib.metadata" -except ImportError: +else: # if the fallback library is missing, we are doomed. - from importlib_metadata import EntryPoint # type: ignore[no-redef] + from importlib_metadata import EntryPoint importlib_metadata_mock = "importlib_metadata"