From 4fef6190b28e8ee55919b2321ea4996c0881fdaf Mon Sep 17 00:00:00 2001 From: zequihg50 Date: Mon, 23 Oct 2023 16:48:35 +0200 Subject: [PATCH 01/10] added driver for h5netcdf --- xarray/backends/h5netcdf_.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 19748084625..7c5b634d951 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -140,6 +140,7 @@ def open( invalid_netcdf=None, phony_dims=None, decode_vlen_strings=True, + driver=None, ): import h5netcdf @@ -161,6 +162,7 @@ def open( kwargs = { "invalid_netcdf": invalid_netcdf, "decode_vlen_strings": decode_vlen_strings, + "driver": driver, } if phony_dims is not None: kwargs["phony_dims"] = phony_dims @@ -397,6 +399,7 @@ def open_dataset( # type: ignore[override] # allow LSP violation, not supporti invalid_netcdf=None, phony_dims=None, decode_vlen_strings=True, + driver=None, ) -> Dataset: filename_or_obj = _normalize_path(filename_or_obj) store = H5NetCDFStore.open( @@ -407,6 +410,7 @@ def open_dataset( # type: ignore[override] # allow LSP violation, not supporti invalid_netcdf=invalid_netcdf, phony_dims=phony_dims, decode_vlen_strings=decode_vlen_strings, + driver=driver, ) store_entrypoint = StoreBackendEntrypoint() From 4ba3bb0a04f814353991802ee48e91535bdcdae5 Mon Sep 17 00:00:00 2001 From: Ezequiel Cimadevilla Alvarez Date: Mon, 13 Nov 2023 09:40:09 +0100 Subject: [PATCH 02/10] Update xarray/backends/h5netcdf_.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Mühlbauer --- xarray/backends/h5netcdf_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 7c5b634d951..9840b6b09e0 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -141,6 +141,7 @@ def open( phony_dims=None, decode_vlen_strings=True, driver=None, + driver_kwds=None, ): import h5netcdf From 6a1db942415be672a4b2ef9f96164b10d4679232 Mon Sep 17 00:00:00 2001 From: Ezequiel Cimadevilla Alvarez Date: Mon, 13 Nov 2023 09:40:33 +0100 Subject: [PATCH 03/10] Update xarray/backends/h5netcdf_.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Mühlbauer --- xarray/backends/h5netcdf_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 9840b6b09e0..1a582cd2e9d 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -401,6 +401,7 @@ def open_dataset( # type: ignore[override] # allow LSP violation, not supporti phony_dims=None, decode_vlen_strings=True, driver=None, + driver_kwds=None, ) -> Dataset: filename_or_obj = _normalize_path(filename_or_obj) store = H5NetCDFStore.open( From 1f2a98e6193dc3320f76c44ee104f7c34375fb9e Mon Sep 17 00:00:00 2001 From: Ezequiel Cimadevilla Alvarez Date: Mon, 13 Nov 2023 09:40:52 +0100 Subject: [PATCH 04/10] Update xarray/backends/h5netcdf_.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Mühlbauer --- xarray/backends/h5netcdf_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 1a582cd2e9d..654f3c4ec7c 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -413,6 +413,7 @@ def open_dataset( # type: ignore[override] # allow LSP violation, not supporti phony_dims=phony_dims, decode_vlen_strings=decode_vlen_strings, driver=driver, + driver_kwds=driver_kwds, ) store_entrypoint = StoreBackendEntrypoint() From 7a4978cc5d7150bf4a3636b07b44be269e4ebfcf Mon Sep 17 00:00:00 2001 From: Ezequiel Cimadevilla Alvarez Date: Mon, 13 Nov 2023 09:41:09 +0100 Subject: [PATCH 05/10] Update xarray/backends/h5netcdf_.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Mühlbauer --- xarray/backends/h5netcdf_.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 654f3c4ec7c..a68a44b5f6f 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -165,6 +165,8 @@ def open( "decode_vlen_strings": decode_vlen_strings, "driver": driver, } + if driver_kwds is not None: + kwargs.update(driver_kwds) if phony_dims is not None: kwargs["phony_dims"] = phony_dims From d00b803f34c1b9c7ed7e62d6d1fb47827442ac1b Mon Sep 17 00:00:00 2001 From: zequihg50 Date: Mon, 13 Nov 2023 09:55:36 +0100 Subject: [PATCH 06/10] added test for h5netcdf ros3 --- xarray/tests/test_backends.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 59e9f655b2e..451552d5531 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -3444,6 +3444,31 @@ def test_write_inconsistent_chunks(self) -> None: assert actual["y"].encoding["chunksizes"] == (100, 50) +@requires_h5netcdf +class TestH5NetCDFDataRos3Driver(TestCommon): + engine: T_NetcdfEngine = "h5netcdf" + test_remote_dataset: str = ( + "https://www.unidata.ucar.edu/software/netcdf/examples/OMI-Aura_L2-example.nc" + ) + + def test_get_variable_list(self) -> None: + with open_dataset(self.test_remote_dataset, engine="h5netcdf", backend_kwargs={"driver": "ros3"}) as actual: + assert "Temperature" in list(actual) + + def test_get_variable_list_empty_driver_kwds(self) -> None: + driver_kwds = { + "secret_id": b"", + "secret_key": b"", + } + backend_kwargs = { + "driver": "ros3", + "driver_kwds": driver_kwds + } + + with open_dataset(self.test_remote_dataset, engine="h5netcdf", backend_kwargs=backend_kwargs) as actual: + assert "Temperature" in list(actual) + + @pytest.fixture(params=["scipy", "netcdf4", "h5netcdf", "pynio", "zarr"]) def readengine(request): return request.param From 5e43a581c583b23af51e0bdb4bfd5fa6820cc723 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:03:02 +0000 Subject: [PATCH 07/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_backends.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 451552d5531..b8de49c0365 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -3452,7 +3452,11 @@ class TestH5NetCDFDataRos3Driver(TestCommon): ) def test_get_variable_list(self) -> None: - with open_dataset(self.test_remote_dataset, engine="h5netcdf", backend_kwargs={"driver": "ros3"}) as actual: + with open_dataset( + self.test_remote_dataset, + engine="h5netcdf", + backend_kwargs={"driver": "ros3"}, + ) as actual: assert "Temperature" in list(actual) def test_get_variable_list_empty_driver_kwds(self) -> None: @@ -3460,12 +3464,11 @@ def test_get_variable_list_empty_driver_kwds(self) -> None: "secret_id": b"", "secret_key": b"", } - backend_kwargs = { - "driver": "ros3", - "driver_kwds": driver_kwds - } + backend_kwargs = {"driver": "ros3", "driver_kwds": driver_kwds} - with open_dataset(self.test_remote_dataset, engine="h5netcdf", backend_kwargs=backend_kwargs) as actual: + with open_dataset( + self.test_remote_dataset, engine="h5netcdf", backend_kwargs=backend_kwargs + ) as actual: assert "Temperature" in list(actual) From e58350ba571f54e0518800e50fd0ef8ab0bda0b8 Mon Sep 17 00:00:00 2001 From: zequihg50 Date: Tue, 14 Nov 2023 10:21:14 +0100 Subject: [PATCH 08/10] check h5netcdf 1.3.0 available for test --- xarray/tests/__init__.py | 4 ++++ xarray/tests/test_backends.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 07ba0be6a8c..0a70334bc01 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -95,6 +95,10 @@ def _importorskip( requires_pandas_version_two = pytest.mark.skipif( not has_pandas_version_two, reason="requires pandas 2.0.0" ) +has_h5netcdf_ros3 = _importorskip("h5netcdf", "1.3.0") +requires_h5netcdf_ros3 = pytest.mark.skipif( + not has_h5netcdf_ros3[0], reason="requires h5netcdf 1.3.0" +) # change some global options for tests set_options(warn_for_unclosed_files=True) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b8de49c0365..b9fa499a03d 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -77,6 +77,7 @@ requires_scipy, requires_scipy_or_netCDF4, requires_zarr, + requires_h5netcdf_ros3, ) from xarray.tests.test_coding_times import ( _ALL_CALENDARS, @@ -3444,7 +3445,7 @@ def test_write_inconsistent_chunks(self) -> None: assert actual["y"].encoding["chunksizes"] == (100, 50) -@requires_h5netcdf +@requires_h5netcdf_ros3 class TestH5NetCDFDataRos3Driver(TestCommon): engine: T_NetcdfEngine = "h5netcdf" test_remote_dataset: str = ( From 633ebed2e8f559ae9500d9e1f28074d14556513b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:42:32 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b9fa499a03d..1a97349ee52 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -69,6 +69,7 @@ requires_dask, requires_fsspec, requires_h5netcdf, + requires_h5netcdf_ros3, requires_iris, requires_netCDF4, requires_pseudonetcdf, @@ -77,7 +78,6 @@ requires_scipy, requires_scipy_or_netCDF4, requires_zarr, - requires_h5netcdf_ros3, ) from xarray.tests.test_coding_times import ( _ALL_CALENDARS, From 190d7d4f539ce72c3caf9ee5245677284bd38c6f Mon Sep 17 00:00:00 2001 From: zequihg50 Date: Wed, 15 Nov 2023 16:40:52 +0100 Subject: [PATCH 10/10] updated doc/whats-new.rst --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5cfec4189b1..5037e4df09e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -30,6 +30,8 @@ New Features region to write in the original store. Also implement automatic transpose when dimension order does not match the original store. (:issue:`7702`, :issue:`8421`, :pull:`8434`). By `Sam Levang `_. +- Allow the usage of h5py drivers (eg: ros3) via h5netcdf (:pull:`8360`). + By `Ezequiel Cimadevilla `_. Breaking changes ~~~~~~~~~~~~~~~~