Merge pull request #358 from CSHS-CWRA/fix-py310

Update dependencies, stabilize test, disable macOS builds

.github/workflows/main.yml

@@ -42,29 +42,29 @@ jobs:
           - tox-env: py39-linux
             python-version: "3.9"
             os: ubuntu-latest
-          - tox-env: py39-macos
-            python-version: "3.9"
-            os: macos-latest
+#          - tox-env: py39-macos
+#            python-version: "3.9"
+#            os: macos-latest
           - tox-env: py310-linux-upstream
             python-version: "3.10"
             os: ubuntu-latest
             upstream-branch: "main"
-          - tox-env: py310-macos-upstream
-            python-version: "3.10"
-            os: macos-latest
-            upstream-branch: "main"
+#          - tox-env: py310-macos-upstream
+#            python-version: "3.10"
+#            os: macos-latest
+#            upstream-branch: "main"
           - tox-env: py311-linux
             python-version: "3.11"
             os: ubuntu-latest
-          - tox-env: py311-macos
-            python-version: "3.11"
-            os: macos-latest
+#          - tox-env: py311-macos
+#            python-version: "3.11"
+#            os: macos-latest
           - tox-env: py312-linux
             python-version: "3.12"
             os: ubuntu-latest
-          - tox-env: py312-macos
-            python-version: "3.12"
-            os: macos-latest
+#          - tox-env: py312-macos
+#            python-version: "3.12"
+#            os: macos-latest
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -112,7 +112,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]  # windows-latest  # disabled until xesmf is available
+        os:
+          - ubuntu-latest
+          # - macos-latest  # disabled until a new build of raven-hydro is available
+          # - windows-latest  # disabled until xesmf is available
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     defaults:
       run:

HISTORY.rst

@@ -2,6 +2,18 @@
 History
 =======
 
+0.15.0 (unreleased)
+-------------------
+* Upgraded `owslib` to `>=0.29.1`. (PR #358)
+* All operations that open NetCDF files or DAP links accept an `engine` argument. The default for all of these is `h5netcdf`. (PR #358)
+* Added `pydap` as an alternate backend for opening DAP links. (PR #358)
+
+Internal changes
+^^^^^^^^^^^^^^^^
+* Added some development dependencies that were missing to the `environment.yml`. (PR #358)
+* `test_climpred_hindcast_verif` is now skipped for Python3.10 builds. It seems to only fail on the particular version of Python. When examining the dependencies, other than the Python version (and ABI version), there are no differences in the environments between Python3.10 and Python3.11. My gut feeling points to an issue with `climpred`. (PR #358)
+* Temporarily disabled tests for macOS on GitHub due to architecture changes. (PR #358)
+
 0.14.0 (2024-03-13)
 -------------------
 * Add support for new processes and methods added in Raven v3.8. (PR #335)

environment-rtd.yml

@@ -23,6 +23,7 @@ dependencies:
   - notebook
   - pandoc
   - pydantic >=2.0
+  - pygments
   - salib
   - seaborn
   - sphinx

environment.yml

@@ -8,6 +8,7 @@ dependencies:
   - libgcc  # for mixing raven-hydro from PyPI with conda environments
   - affine
   - black >=24.2.0
+  - bump2version >=1.0.1
   - cftime
   - cf_xarray
   - click
@@ -25,25 +26,32 @@ dependencies:
   - isort >=5.13.2
   - lxml
   - matplotlib
+  - mypy
   - netcdf4
   - numpy
-  - owslib <0.29.0  # see: https://github.com/geopython/OWSLib/issues/871
+  - owslib >=0.29.1
   - pandas >=2.2.0
   - pint >=0.20
   - platformdirs
   - pre-commit
   - pydantic >=2.0
+  - pydap
   - pymbolic
   - pyogrio
   - pyproj >=3.0
+  - pytest
+  - pytest-cov
+  - pytest-xdist >=3.2.0
   - rasterio
   - requests
   - rioxarray
   - scipy
   - shapely
   - spotpy
   - statsmodels
+  - tox >=4.5
   - typing_extensions
+  - watchdog
   - xarray >=2023.11.0  # xarray v2023.9.0 is incompatible with xclim<=0.45.0
   - xclim >=0.48.2
   - xesmf

pyproject.toml

@@ -37,19 +37,20 @@ dynamic = ["description", "version"]
 dependencies = [
   "cftime",
   # cf-xarray is differently named on conda-forge
-  "cf-xarray[all]",
+  "cf-xarray",
   "climpred >=2.4.0",
   "dask",
   "haversine",
   "h5netcdf",
   "matplotlib",
   "netCDF4",
   "numpy",
-  "owslib >=0.24.1,<0.29", # see: https://github.com/geopython/OWSLib/issues/871
+  "owslib >=0.29.1",
   "pandas >=2.2.0",
   "pint >=0.20",
   "platformdirs",
   "pydantic >=2.0",
+  "pydap",
   "pymbolic",
   "raven-hydro >=0.2.4,<1.0",
   "requests",
@@ -80,7 +81,7 @@ dev = [
   "pytest-cov",
   "pytest-xdist >=3.2.0",
   "setuptools >=68.0",
-  "tox",
+  "tox >=4.5",
   "watchdog",
   "wheel >=0.42.0"
 ]
@@ -106,6 +107,7 @@ docs = [
   "myst_nb",
   "nbsphinx",
   "numpydoc",
+  "pygments",
   "pymetalink",
   "salib",
   "s3fs",

ravenpy/config/commands.py

@@ -555,9 +555,17 @@ def reorder_time(cls, v):
         return tuple(dims)
 
     @classmethod
-    def from_nc(cls, fn, data_type, station_idx=1, alt_names=(), **kwds):
+    def from_nc(
+        cls, fn, data_type, station_idx=1, alt_names=(), engine="h5netcdf", **kwds
+    ):
         """Instantiate class from netCDF dataset."""
-        specs = nc_specs(fn, data_type, station_idx, alt_names)
+        specs = nc_specs(
+            fn,
+            data_type,
+            station_idx=station_idx,
+            alt_names=alt_names,
+            engine=engine,
+        )
         specs.update(kwds)
         attrs = filter_for(cls, specs)
         return cls(**attrs)
@@ -566,11 +574,7 @@ def from_nc(cls, fn, data_type, station_idx=1, alt_names=(), **kwds):
     def da(self) -> xr.DataArray:
         """Return DataArray from configuration."""
         # TODO: Apply linear transform and time shift
-        # FIXME: Workaround for macOS bug
-        try:
-            da = xr.open_dataset(self.file_name_nc)[self.var_name_nc]
-        except ValueError:
-            da = xr.open_dataset(self.file_name_nc, engine="h5netcdf")[self.var_name_nc]
+        da = xr.open_dataset(self.file_name_nc)[self.var_name_nc]
         if len(self.dim_names_nc) == 1:
             return da
         elif len(self.dim_names_nc) == 2:
@@ -714,20 +718,21 @@ def confirm_monthly(cls, v):
     def from_nc(
         cls,
         fn: Union[str, Path, Sequence[Path]],
-        data_type: Sequence[str] = None,
+        data_type: Optional[Sequence[str]] = None,
         station_idx: int = 1,
         alt_names: Optional[Dict[str, str]] = None,
         mon_ave: bool = False,
         data_kwds: Optional[Dict[str, Any]] = None,
+        engine: str = "h5netcdf",
         **kwds,
     ) -> "Gauge":
         """Return Gauge instance with configuration options inferred from the netCDF itself.
 
         Parameters
         ----------
-        fn : Union[str, Path, Sequence[Path]],
+        fn : str or Path or Sequence[Path]
             NetCDF file path or paths.
-        data_type : Sequence[str], None
+        data_type : Sequence[str], optional
             Raven data types to extract from netCDF files, e.g. 'PRECIP', 'AVE_TEMP'. The algorithm tries to find all
             forcings in each file until one is found, then it stops searching for it in the following files.
         station_idx : int
@@ -737,9 +742,11 @@ def from_nc(
             Use this if variables do not correspond to CF standard defaults.
         mon_ave : bool
             If True, compute the monthly average.
-        data_kwds : Dict[options.Forcings, Dict[str, str]]]
+        data_kwds : dict[options.Forcings, dict[str, str]]
             Additional `:Data` parameters keyed by forcing type and station id. Overrides inferred parameters.
             Use keyword "ALL" to pass parameters to all variables.
+        engine : {"h5netcdf", "netcdf4", "pydap"}
+            The engine used to open the dataset. Default is 'h5netcdf'.
         **kwds
             Additional arguments for Gauge.
 
@@ -766,7 +773,12 @@ def from_nc(
             for dtype in forcings:
                 try:
                     specs = nc_specs(
-                        f, dtype, idx, alt_names.get(dtype, ()), mon_ave=mon_ave
+                        f,
+                        dtype,
+                        idx,
+                        alt_names.get(dtype, ()),
+                        mon_ave=mon_ave,
+                        engine=engine,
                     )
                 except ValueError:
                     pass
@@ -814,8 +826,14 @@ def _template(self):
         """
 
     @classmethod
-    def from_nc(cls, fn, station_idx: int = 1, alt_names=(), **kwds):
-        specs = nc_specs(fn, "HYDROGRAPH", station_idx, alt_names)
+    def from_nc(cls, fn, station_idx: int = 1, alt_names=(), engine="h5netcdf", **kwds):
+        specs = nc_specs(
+            fn,
+            "HYDROGRAPH",
+            station_idx=station_idx,
+            alt_names=alt_names,
+            engine=engine,
+        )
         attrs = filter_for(cls, specs, **kwds, data_type="HYDROGRAPH")
         return cls(**attrs)
 

ravenpy/config/utils.py

@@ -11,11 +11,12 @@
 
 
 def nc_specs(
-    fn: Union[str, os.PathLike],
+    fn: Union[str, os.PathLike[str]],
     data_type: str,
     station_idx: Optional[int] = None,
     alt_names: Union[str, Sequence[str]] = None,
     mon_ave: bool = False,
+    engine: str = "h5netcdf",
     # FIXME: Is this call signature still relevant?
     linear_transform=None,
 ):
@@ -33,6 +34,8 @@ def nc_specs(
         Alternative variable names for data type if not the CF standard default.
     mon_ave : bool
         If True, compute the monthly average.
+    engine : str
+        The engine used to open the dataset. Default is 'h5netcdf'.
 
     Returns
     -------
@@ -44,18 +47,16 @@ def nc_specs(
     elevation_var_name_nc
     latitude, longitude, elevation, name
     """
-    from pathlib import Path
-
     from ravenpy.utilities.coords import infer_scale_and_offset
 
-    # Convert to NumPy 0-based indexing
-    if station_idx is not None:
-        i = station_idx - 1
-
     if isinstance(fn, str) and str(fn)[:4] == "http":
         pass
-    elif Path(fn).exists():
-        fn = Path(fn).resolve(strict=True)
+    elif os.path.exists(fn):
+        # `strict` kwarg is not available in Python 3.9
+        try:
+            fn = os.path.realpath(fn, strict=True)
+        except TypeError:
+            fn = os.path.realpath(fn)
     else:
         raise ValueError("NetCDF file not found.")
 
@@ -69,7 +70,7 @@ def nc_specs(
         "station_idx": station_idx,
     }
 
-    with xr.open_dataset(fn) as ds:
+    with xr.open_dataset(fn, engine=engine) as ds:
         var_names = CF_RAVEN.get(data_type, ()) + tuple(alt_names)
         if len(var_names) == 0:
             raise ValueError(
@@ -97,6 +98,9 @@ def nc_specs(
             raise ValueError(f"No variable found for {data_type}.\n {ds.data_vars}")
 
         if station_idx is not None:
+            # Convert to NumPy 0-based indexing
+            i = station_idx - 1
+
             try:
                 attrs["latitude_var_name_nc"] = ds.cf["latitude"].name
                 attrs["longitude_var_name_nc"] = ds.cf["longitude"].name
@@ -125,7 +129,7 @@ def nc_specs(
                 if ds["station_id"].shape and len(ds["station_id"]) > i:
                     attrs["name"] = ds["station_id"].values[i]
 
-        return attrs
+    return attrs
 
 
 def filter_for(kls, attrs, **kwds):
@@ -163,7 +167,7 @@ def filter_for(kls, attrs, **kwds):
 
 
 def get_average_annual_runoff(
-    nc_file_path: Union[str, os.PathLike],
+    nc_file_path: Union[str, os.PathLike[str]],
     area_in_m2: float,
     time_dim: str = "time",
     obs_var: str = "qobs",

tests/test_commands.py

@@ -351,11 +351,11 @@ def test_gauge(get_local_testdata, tmp_path):
     salmon_file = get_local_testdata(
         "raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
     )
-
-    copyfile(salmon_file, tmp_path / "Salmon-River-Near-Prince-George_meteo_daily.nc")
+    salmon_file_tmp = tmp_path / "salmon_river_near_prince_george-tmp.nc"
+    salmon_file_tmp.write_bytes(salmon_file.read_bytes())
 
     g = rc.Gauge.from_nc(
-        tmp_path / "Salmon-River-Near-Prince-George_meteo_daily.nc",
+        salmon_file_tmp,
         alt_names={"RAINFALL": "rain", "SNOWFALL": "snow"},
         data_kwds={"ALL": {"Deaccumulate": True}},
     )

tests/test_emulators.py

@@ -666,9 +666,7 @@ def test_canopex():
 
     qobs = [
         rc.ObservationData.from_nc(
-            CANOPEX_DAP,
-            alt_names="discharge",
-            station_idx=basin,
+            CANOPEX_DAP, alt_names="discharge", station_idx=basin, engine="netcdf4"
         )
     ]
 
@@ -678,6 +676,7 @@ def test_canopex():
             station_idx=basin,
             data_type=data_type,  # Note that this is the list of all the variables
             alt_names=alt_names,  # Note that all variables here are mapped to their names in the netcdf file.
+            engine="netcdf4",
             data_kwds=data_kwds,
         )
     ]