Merge branch 'main' into zarr-v3-b2

* main: Bump codecov/codecov-action from 4.6.0 to 5.0.2 in the actions group (pydata#9793) Buffer types (pydata#9787)
dcherian · Nov 18, 2024 · aa522e7 · aa522e7
2 parents c07ed88 + 3f0ddc1
commit aa522e7
Show file tree

Hide file tree

Showing 14 changed files with 129 additions and 73 deletions.
diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -123,7 +123,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: mypy_report/cobertura.xml
           flags: mypy
@@ -174,7 +174,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: mypy_report/cobertura.xml
           flags: mypy-min
@@ -230,7 +230,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: pyright_report/cobertura.xml
           flags: pyright
@@ -286,7 +286,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: pyright_report/cobertura.xml
           flags: pyright39

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -159,7 +159,7 @@ jobs:
           path: pytest.xml
 
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: ./coverage.xml
           flags: unittests

diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -140,7 +140,7 @@ jobs:
         run: |
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.2
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

diff --git a/xarray/backends/api.py b/xarray/backends/api.py
@@ -54,14 +54,14 @@
         from dask.delayed import Delayed
     except ImportError:
         Delayed = None  # type: ignore[assignment, misc]
-    from io import BufferedIOBase
 
     from xarray.backends.common import BackendEntrypoint
     from xarray.core.types import (
         CombineAttrsOptions,
         CompatOptions,
         JoinOptions,
         NestedSequence,
+        ReadBuffer,
         T_Chunks,
     )
 
@@ -474,7 +474,7 @@ def _datatree_from_backend_datatree(
 
 
 def open_dataset(
-    filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     *,
     engine: T_Engine = None,
     chunks: T_Chunks = None,
@@ -691,7 +691,7 @@ def open_dataset(
 
 
 def open_dataarray(
-    filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     *,
     engine: T_Engine | None = None,
     chunks: T_Chunks | None = None,
@@ -896,7 +896,7 @@ def open_dataarray(
 
 
 def open_datatree(
-    filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     *,
     engine: T_Engine = None,
     chunks: T_Chunks = None,
@@ -1111,7 +1111,7 @@ def open_datatree(
 
 
 def open_groups(
-    filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     *,
     engine: T_Engine = None,
     chunks: T_Chunks = None,
@@ -1137,10 +1137,6 @@ def open_groups(
     and cannot be opened directly with ``open_datatree``. It is encouraged to use this function to inspect your data,
     then make the necessary changes to make the structure coercible to a `DataTree` object before calling `DataTree.from_dict()` and proceeding with your analysis.
 
-    Parameters
-    ----------
-    filename_or_obj : str, Path, file-like, or DataStore
-        Strings and Path objects are interpreted as a path to a netCDF file.
     Parameters
     ----------
     filename_or_obj : str, Path, file-like, or DataStore
@@ -1338,7 +1334,10 @@ def open_groups(
 
 
 def open_mfdataset(
-    paths: str | os.PathLike | NestedSequence[str | os.PathLike],
+    paths: str
+    | os.PathLike
+    | ReadBuffer
+    | NestedSequence[str | os.PathLike | ReadBuffer],
     chunks: T_Chunks | None = None,
     concat_dim: (
         str
@@ -1541,7 +1540,7 @@ def open_mfdataset(
     if not paths:
         raise OSError("no files to open")
 
-    paths1d: list[str]
+    paths1d: list[str | ReadBuffer]
     if combine == "nested":
         if isinstance(concat_dim, str | DataArray) or concat_dim is None:
             concat_dim = [concat_dim]  # type: ignore[assignment]

diff --git a/xarray/backends/common.py b/xarray/backends/common.py
@@ -6,20 +6,19 @@
 import traceback
 from collections.abc import Iterable, Mapping, Sequence
 from glob import glob
-from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, overload
+from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload
 
 import numpy as np
 
 from xarray.conventions import cf_encoder
 from xarray.core import indexing
 from xarray.core.datatree import DataTree
+from xarray.core.types import ReadBuffer
 from xarray.core.utils import FrozenDict, NdimSizeLenMixin, is_remote_uri
 from xarray.namedarray.parallelcompat import get_chunked_array_type
 from xarray.namedarray.pycompat import is_chunked_array
 
 if TYPE_CHECKING:
-    from io import BufferedIOBase
-
     from xarray.core.dataset import Dataset
     from xarray.core.types import NestedSequence
 
@@ -65,24 +64,52 @@ def _normalize_path(path: str | os.PathLike | T) -> str | T:
     if isinstance(path, str) and not is_remote_uri(path):
         path = os.path.abspath(os.path.expanduser(path))
 
-    return cast(str, path)
+    return path  # type:ignore [return-value]
 
 
 @overload
 def _find_absolute_paths(
-    paths: str | os.PathLike | Sequence[str | os.PathLike], **kwargs
+    paths: str | os.PathLike | Sequence[str | os.PathLike],
+    **kwargs,
 ) -> list[str]: ...
 
 
+@overload
+def _find_absolute_paths(
+    paths: ReadBuffer | Sequence[ReadBuffer],
+    **kwargs,
+) -> list[ReadBuffer]: ...
+
+
 @overload
 def _find_absolute_paths(
     paths: NestedSequence[str | os.PathLike], **kwargs
 ) -> NestedSequence[str]: ...
 
 
+@overload
 def _find_absolute_paths(
-    paths: str | os.PathLike | NestedSequence[str | os.PathLike], **kwargs
-) -> NestedSequence[str]:
+    paths: NestedSequence[ReadBuffer], **kwargs
+) -> NestedSequence[ReadBuffer]: ...
+
+
+@overload
+def _find_absolute_paths(
+    paths: str
+    | os.PathLike
+    | ReadBuffer
+    | NestedSequence[str | os.PathLike | ReadBuffer],
+    **kwargs,
+) -> NestedSequence[str | ReadBuffer]: ...
+
+
+def _find_absolute_paths(
+    paths: str
+    | os.PathLike
+    | ReadBuffer
+    | NestedSequence[str | os.PathLike | ReadBuffer],
+    **kwargs,
+) -> NestedSequence[str | ReadBuffer]:
     """
     Find absolute paths from the pattern.
 
@@ -132,10 +159,12 @@ def _find_absolute_paths(
             return sorted(glob(_normalize_path(paths)))
     elif isinstance(paths, os.PathLike):
         return [_normalize_path(paths)]
+    elif isinstance(paths, ReadBuffer):
+        return [paths]
 
     def _normalize_path_list(
-        lpaths: NestedSequence[str | os.PathLike],
-    ) -> NestedSequence[str]:
+        lpaths: NestedSequence[str | os.PathLike | ReadBuffer],
+    ) -> NestedSequence[str | ReadBuffer]:
         paths = []
         for p in lpaths:
             if isinstance(p, str | os.PathLike):
@@ -546,10 +575,9 @@ def __repr__(self) -> str:
 
     def open_dataset(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         drop_variables: str | Iterable[str] | None = None,
-        **kwargs: Any,
     ) -> Dataset:
         """
         Backend open_dataset method used by Xarray in :py:func:`~xarray.open_dataset`.
@@ -559,7 +587,7 @@ def open_dataset(
 
     def guess_can_open(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     ) -> bool:
         """
         Backend open_dataset method used by Xarray in :py:func:`~xarray.open_dataset`.
@@ -569,8 +597,9 @@ def guess_can_open(
 
     def open_datatree(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
-        **kwargs: Any,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
+        *,
+        drop_variables: str | Iterable[str] | None = None,
     ) -> DataTree:
         """
         Backend open_datatree method used by Xarray in :py:func:`~xarray.open_datatree`.
@@ -580,8 +609,9 @@ def open_datatree(
 
     def open_groups_as_dict(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
-        **kwargs: Any,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
+        *,
+        drop_variables: str | Iterable[str] | None = None,
     ) -> dict[str, Dataset]:
         """
         Opens a dictionary mapping from group names to Datasets.

diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py
@@ -39,11 +39,10 @@
 from xarray.core.variable import Variable
 
 if TYPE_CHECKING:
-    from io import BufferedIOBase
-
     from xarray.backends.common import AbstractDataStore
     from xarray.core.dataset import Dataset
     from xarray.core.datatree import DataTree
+    from xarray.core.types import ReadBuffer
 
 
 class H5NetCDFArrayWrapper(BaseNetCDF4Array):
@@ -395,7 +394,7 @@ class H5netcdfBackendEntrypoint(BackendEntrypoint):
 
     def guess_can_open(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     ) -> bool:
         magic_number = try_read_magic_number_from_file_or_path(filename_or_obj)
         if magic_number is not None:
@@ -407,9 +406,9 @@ def guess_can_open(
 
         return False
 
-    def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporting **kwargs
+    def open_dataset(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,
@@ -456,7 +455,7 @@ def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporti
 
     def open_datatree(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,
@@ -499,7 +498,7 @@ def open_datatree(
 
     def open_groups_as_dict(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,

diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py
@@ -41,14 +41,13 @@
 from xarray.core.variable import Variable
 
 if TYPE_CHECKING:
-    from io import BufferedIOBase
-
     from h5netcdf.core import EnumType as h5EnumType
     from netCDF4 import EnumType as ncEnumType
 
     from xarray.backends.common import AbstractDataStore
     from xarray.core.dataset import Dataset
     from xarray.core.datatree import DataTree
+    from xarray.core.types import ReadBuffer
 
 # This lookup table maps from dtype.byteorder to a readable endian
 # string used by netCDF4.
@@ -627,7 +626,7 @@ class NetCDF4BackendEntrypoint(BackendEntrypoint):
 
     def guess_can_open(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     ) -> bool:
         if isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj):
             return True
@@ -642,9 +641,9 @@ def guess_can_open(
 
         return False
 
-    def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporting **kwargs
+    def open_dataset(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,
@@ -693,7 +692,7 @@ def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporti
 
     def open_datatree(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,
@@ -735,7 +734,7 @@ def open_datatree(
 
     def open_groups_as_dict(
         self,
-        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,