Skip to content

Commit

Permalink
Explicitly specify HDF5VirtualBackend for test parameter.
Browse files Browse the repository at this point in the history
  • Loading branch information
sharkinsspatial committed Nov 15, 2024
1 parent 3e216dc commit 5b085a6
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 21 deletions.
29 changes: 13 additions & 16 deletions virtualizarr/tests/test_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from virtualizarr import open_virtual_dataset
from virtualizarr.backend import FileType, automatically_determine_filetype
from virtualizarr.manifests import ManifestArray
from virtualizarr.readers import HDF5VirtualBackend
from virtualizarr.readers.hdf import HDFVirtualBackend
from virtualizarr.tests import (
has_astropy,
Expand Down Expand Up @@ -83,7 +84,7 @@ def test_FileType():


@requires_kerchunk
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
class TestOpenVirtualDatasetIndexes:
def test_no_indexes(self, netcdf4_file, hdf_backend):
vds = open_virtual_dataset(netcdf4_file, indexes={}, backend=hdf_backend)
Expand Down Expand Up @@ -115,7 +116,7 @@ def index_mappings_equal(indexes1: Mapping[str, Index], indexes2: Mapping[str, I


@requires_kerchunk
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_cftime_index(tmpdir, hdf_backend):
"""Ensure a virtual dataset contains the same indexes as an Xarray dataset"""
# Note: Test was created to debug: https://github.com/zarr-developers/VirtualiZarr/issues/168
Expand Down Expand Up @@ -145,7 +146,7 @@ def test_cftime_index(tmpdir, hdf_backend):


@requires_kerchunk
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
class TestOpenVirtualDatasetAttrs:
def test_drop_array_dimensions(self, netcdf4_file, hdf_backend):
# regression test for GH issue #150
Expand All @@ -166,20 +167,16 @@ def test_coordinate_variable_attrs_preserved(self, netcdf4_file, hdf_backend):
@network
@requires_s3fs
class TestReadFromS3:
@pytest.mark.parametrize(
"filetype", ["netcdf4", None], ids=["netcdf4 filetype", "None filetype"]
)
@pytest.mark.parametrize(
"indexes", [None, {}], ids=["None index", "empty dict index"]
)
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
def test_anon_read_s3(self, filetype, indexes, hdf_backend):
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_anon_read_s3(self, indexes, hdf_backend):
"""Parameterized tests for empty vs supplied indexes and filetypes."""
# TODO: Switch away from this s3 url after minIO is implemented.
fpath = "s3://carbonplan-share/virtualizarr/local.nc"
vds = open_virtual_dataset(
fpath,
filetype=filetype,
indexes=indexes,
reader_options={"storage_options": {"anon": True}},
backend=hdf_backend,
Expand All @@ -191,7 +188,7 @@ def test_anon_read_s3(self, filetype, indexes, hdf_backend):


@network
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
class TestReadFromURL:
@pytest.mark.parametrize(
"filetype, url",
Expand Down Expand Up @@ -295,7 +292,7 @@ def test_virtualizarr_vs_local_nisar(self, hdf_backend):

@requires_kerchunk
class TestLoadVirtualDataset:
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_loadable_variables(self, netcdf4_file, hdf_backend):
vars_to_load = ["air", "time"]
vds = open_virtual_dataset(
Expand Down Expand Up @@ -330,16 +327,16 @@ def test_explicit_filetype_and_backend(self, netcdf4_file):
netcdf4_file, filetype="hdf", backend=HDFVirtualBackend
)

@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_group_kwarg(self, hdf5_groups_file, hdf_backend):
if hdf_backend:
if hdf_backend == HDFVirtualBackend:
with pytest.raises(NotImplementedError, match="Nested groups"):
open_virtual_dataset(hdf5_groups_file, backend=hdf_backend)
with pytest.raises(KeyError, match="doesn't exist"):
open_virtual_dataset(
hdf5_groups_file, group="doesnt_exist", backend=hdf_backend
)
else:
if hdf_backend == HDF5VirtualBackend:
with pytest.raises(ValueError, match="Multiple HDF Groups found"):
open_virtual_dataset(hdf5_groups_file)
with pytest.raises(ValueError, match="not found in"):
Expand Down Expand Up @@ -376,13 +373,13 @@ def test_open_virtual_dataset_passes_expected_args(
}
mock_read_kerchunk.assert_called_once_with(**args)

@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_open_dataset_with_empty(self, hdf5_empty, tmpdir, hdf_backend):
vds = open_virtual_dataset(hdf5_empty, backend=hdf_backend)
assert vds.empty.dims == ()
assert vds.empty.attrs == {"empty": "true"}

@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_open_dataset_with_scalar(self, hdf5_scalar, tmpdir, hdf_backend):
vds = open_virtual_dataset(hdf5_scalar, backend=hdf_backend)
assert vds.scalar.dims == ()
Expand Down
11 changes: 6 additions & 5 deletions virtualizarr/tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from virtualizarr import open_virtual_dataset
from virtualizarr.manifests import ChunkManifest, ManifestArray
from virtualizarr.readers import HDF5VirtualBackend
from virtualizarr.readers.hdf import HDFVirtualBackend
from virtualizarr.tests import requires_kerchunk
from virtualizarr.translators.kerchunk import (
Expand Down Expand Up @@ -64,7 +65,7 @@ def test_no_duplicates_find_var_names():
),
],
)
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_numpy_arrays_to_inlined_kerchunk_refs(
netcdf4_file, inline_threshold, vars_to_inline, hdf_backend
):
Expand Down Expand Up @@ -92,7 +93,7 @@ def test_numpy_arrays_to_inlined_kerchunk_refs(
@requires_kerchunk
@pytest.mark.parametrize("format", ["dict", "json", "parquet"])
class TestKerchunkRoundtrip:
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_kerchunk_roundtrip_no_concat(self, tmpdir, format, hdf_backend):
# set up example xarray dataset
ds = xr.tutorial.open_dataset("air_temperature", decode_times=False)
Expand Down Expand Up @@ -125,7 +126,7 @@ def test_kerchunk_roundtrip_no_concat(self, tmpdir, format, hdf_backend):
for coord in ds.coords:
assert ds.coords[coord].attrs == roundtrip.coords[coord].attrs

@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
@pytest.mark.parametrize("decode_times,time_vars", [(False, []), (True, ["time"])])
def test_kerchunk_roundtrip_concat(
self, tmpdir, format, hdf_backend, decode_times, time_vars
Expand Down Expand Up @@ -195,7 +196,7 @@ def test_kerchunk_roundtrip_concat(
assert roundtrip.time.encoding["units"] == ds.time.encoding["units"]
assert roundtrip.time.encoding["calendar"] == ds.time.encoding["calendar"]

@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_non_dimension_coordinates(self, tmpdir, format, hdf_backend):
# regression test for GH issue #105

Expand Down Expand Up @@ -282,7 +283,7 @@ def test_datetime64_dtype_fill_value(self, tmpdir, format):


@requires_kerchunk
@pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend])
@pytest.mark.parametrize("hdf_backend", [HDF5VirtualBackend, HDFVirtualBackend])
def test_open_scalar_variable(tmpdir, hdf_backend):
# regression test for GH issue #100

Expand Down

0 comments on commit 5b085a6

Please sign in to comment.