From 864b35a12ae890af540f35faee258df8e67d1e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Fri, 15 Nov 2024 10:28:52 +0100 Subject: [PATCH 1/2] add 'User-Agent'-header to pooch.retrieve (#9782) * add 'User-Agent'-header to pooch.retrieve * try sys.modules * Apply suggestions from code review Co-authored-by: Mathias Hauser * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add whats-new.rst entry --------- Co-authored-by: Mathias Hauser Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/whats-new.rst | 2 ++ xarray/tutorial.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b74b0fb84de..ee826e6e56f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -64,6 +64,8 @@ Bug fixes By `Pascal Bourgault `_. - Fix CF decoding of ``grid_mapping`` to allow all possible formats, add tests (:issue:`9761`, :pull:`9765`). By `Kai Mühlbauer `_. +- Add `User-Agent` to request-headers when retrieving tutorial data (:issue:`9774`, :pull:`9782`) + By `Kai Mühlbauer `_. Documentation ~~~~~~~~~~~~~ diff --git a/xarray/tutorial.py b/xarray/tutorial.py index ccdb0d8e031..9a5d52ed285 100644 --- a/xarray/tutorial.py +++ b/xarray/tutorial.py @@ -10,6 +10,7 @@ import os import pathlib +import sys from typing import TYPE_CHECKING import numpy as np @@ -157,8 +158,13 @@ def open_dataset( url = f"{base_url}/raw/{version}/{path.name}" + headers = {"User-Agent": f"xarray {sys.modules['xarray'].__version__}"} + downloader = pooch.HTTPDownloader(headers=headers) + # retrieve the file - filepath = pooch.retrieve(url=url, known_hash=None, path=cache_dir) + filepath = pooch.retrieve( + url=url, known_hash=None, path=cache_dir, downloader=downloader + ) ds = _open_dataset(filepath, engine=engine, **kws) if not cache: ds = ds.load() From 9f459c3f4eb38a016966c6dce64bd07ec44c06ab Mon Sep 17 00:00:00 2001 From: Patrick Hoefler <61934744+phofl@users.noreply.github.com> Date: Fri, 15 Nov 2024 21:19:02 +0100 Subject: [PATCH 2/2] Fix open_mfdataset for list of fsspec files (#9785) * Fix open_mfdataset for list of fsspec files * Rewrite to for loop * Fixup --- xarray/backends/common.py | 17 +++++++++-------- xarray/tests/test_backends.py | 2 ++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/xarray/backends/common.py b/xarray/backends/common.py index c51cf3d1964..534567221b5 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -136,14 +136,15 @@ def _find_absolute_paths( def _normalize_path_list( lpaths: NestedSequence[str | os.PathLike], ) -> NestedSequence[str]: - return [ - ( - _normalize_path(p) - if isinstance(p, str | os.PathLike) - else _normalize_path_list(p) - ) - for p in lpaths - ] + paths = [] + for p in lpaths: + if isinstance(p, str | os.PathLike): + paths.append(_normalize_path(p)) + elif isinstance(p, list): + paths.append(_normalize_path_list(p)) # type: ignore[arg-type] + else: + paths.append(p) # type: ignore[arg-type] + return paths return _normalize_path_list(paths) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 7be6eb5ed0d..c543333c61e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -5511,6 +5511,8 @@ def test_source_encoding_always_present_with_fsspec() -> None: fs = fsspec.filesystem("file") with fs.open(tmp) as f, open_dataset(f) as ds: assert ds.encoding["source"] == tmp + with fs.open(tmp) as f, open_mfdataset([f]) as ds: + assert "foo" in ds def _assert_no_dates_out_of_range_warning(record):