Merge branch 'develop'

.coveragerc

@@ -1,8 +1,9 @@
 # .coveragerc to control coverage.py
 
 [run]
-# Also report branch coverage
-branch = True
+# skip test files
+omit =
+    */test*/*
 # Set concurrency type for correct coverage of multi-processing code
 concurrency = multiprocessing
 
@@ -14,12 +15,3 @@ source = climada/
 exclude_also =
     # Main code is not run
     if __name__ == .__main__.:
-
-    # Abtract methods are not run
-    @(abc\.)?abstractmethod
-
-# Never fail when reporting
-ignore_errors = True
-
-[html]
-directory = coverage

.github/workflows/ci.yml

@@ -6,11 +6,11 @@ on: [push]
 # Use bash explicitly for being able to enter the conda environment
 defaults:
   run:
-    shell: bash -l {0}
+    shell: bash -el {0}
 
 jobs:
   build-and-test:
-    name: Build Env, Install, Unit Tests
+    name: 'Core / Unit Test Pipeline'
     runs-on: ubuntu-latest
     permissions:
       # For publishing results
@@ -41,8 +41,6 @@ jobs:
           create-args: >-
             python=${{ matrix.python-version }}
             make
-          init-shell: >-
-            bash
           # Persist environment for branch, Python version, single day
           cache-environment-key: env-${{ github.ref }}-${{ matrix.python-version }}-${{ steps.date.outputs.date }}
       -
@@ -59,12 +57,22 @@ jobs:
         if: always()
         with:
           junit_files: tests_xml/tests.xml
-          check_name: "Unit Test Results Python ${{ matrix.python-version }}"
+          check_name: "Core / Unit Test Results (${{ matrix.python-version }})"
           comment_mode: "off"
       -
         name: Upload Coverage Reports
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-report-unittests-py${{ matrix.python-version }}
+          name: coverage-report-core-unittests-py${{ matrix.python-version }}
           path: coverage/
+
+  test-petals:
+    name: Petals Compatibility
+    uses: CLIMADA-project/climada_petals/.github/workflows/testing.yml@develop
+    needs: build-and-test
+    with:
+      core_branch: ${{ github.ref }}
+      petals_branch: develop
+    permissions:
+      checks: write

.readthedocs.yml

@@ -5,8 +5,13 @@ build:
   tools:
     python: "mambaforge-4.10"
 
+  # Append fixed Python version to requirements
+  jobs:
+    pre_create_environment:
+      - echo "  - python=3.9" >> requirements/env_climada.yml
+
 conda:
-  environment: requirements/env_docs.yml
+  environment: requirements/env_climada.yml
 
 python:
   install:

AUTHORS.md

@@ -29,7 +29,10 @@
 * Raphael Portmann
 * Nicolas Colombi
 * Leonie Villiger
+* Timo Schmid
 * Kam Lam Yeung
 * Sarah Hülsen
 * Timo Schmid
+* Luca Severino
 * Samuel Juhel
+* Valentin Gebhart

CHANGELOG.md

@@ -1,5 +1,118 @@
 # Changelog
 
+## Unreleased
+
+Release date: YYYY-MM-DD
+
+Code freeze date: YYYY-MM-DD
+
+### Description
+
+### Dependency Changes
+
+Added:
+
+- `bayesian-optimization`
+- `seaborn` >=0.13
+
+Updated:
+
+- `bottleneck` >=1.3 → >=1.4
+- `cartopy` >=0.22 → >=0.23
+- `contextily` >=1.5 → >=1.6
+- `dask` >=2024.1,<2024.3 &rarr; >=2024.2,<2024.3
+- `matplotlib-base` >=3.8 &rarr; >=3.9
+- `numba` >=0.59 &rarr; >=0.60
+- `numexpr` >=2.9 &rarr; >=2.10
+- `pint` >=0.23 &rarr; >=0.24
+- `pycountry` >=22.3 &rarr; >=24.6
+- `requests` >=2.31 &rarr; >=2.32
+- `salib` >=1.4 &rarr; >=1.5
+- `scikit-learn` >=1.4 &rarr; >=1.5
+- `scipy` >=1.12 &rarr; >=1.13
+- `xarray` >=2024.2 &rarr; >=2024.6
+
+### Added
+
+- GitHub actions workflow for CLIMADA Petals compatibility tests [#855](https://github.com/CLIMADA-project/climada_python/pull/855)
+- `climada.util.calibrate` module for calibrating impact functions [#692](https://github.com/CLIMADA-project/climada_python/pull/692)
+
+### Changed
+
+- Use Geopandas GeoDataFrame.plot() for centroids plotting function [896](https://github.com/CLIMADA-project/climada_python/pull/896)
+- Update SALib sensitivity and sampling methods from newest version (SALib 1.4.7) [#828](https://github.com/CLIMADA-project/climada_python/issues/828)
+- Allow for computation of relative and absolute delta impacts in `CalcDeltaClimate`
+- Remove content tables and make minor improvements (fix typos and readability) in
+CLIMADA tutorials. [#872](https://github.com/CLIMADA-project/climada_python/pull/872)
+- Centroids complete overhaul. Most function should be backward compatible. Internal data is stored in a geodataframe attribute. Raster are now stored as points, and the meta attribute is removed. Several methds were deprecated or removed. [#787](https://github.com/CLIMADA-project/climada_python/pull/787)
+- Improved error messages produced by `ImpactCalc.impact()` in case impact function in the exposures is not found in impf_set [#863](https://github.com/CLIMADA-project/climada_python/pull/863)
+- Update the Holland et al. 2010 TC windfield model and introduce `model_kwargs` parameter to adjust model parameters [#846](https://github.com/CLIMADA-project/climada_python/pull/846)
+- Changed module structure: `climada.hazard.Hazard` has been split into the modules `base`, `io` and `plot` [#871](https://github.com/CLIMADA-project/climada_python/pull/871)
+- Ensure `csr_matrix` stored in `climada.hazard.Hazard` have consistent data format and store no explicit zeros when initializing `ImpactCalc` [#893](https://github.com/CLIMADA-project/climada_python/pull/893)
+- `Impact.from_hdf5` now calls `str` on `event_name` data that is not strings, and issue a warning then [#894](https://github.com/CLIMADA-project/climada_python/pull/894)
+- `Impact.write_hdf5` now throws an error if `event_name` is does not contain strings exclusively [#894](https://github.com/CLIMADA-project/climada_python/pull/894)
+- Split `climada.hazard.trop_cyclone` module into smaller submodules without affecting module usage [#911](https://github.com/CLIMADA-project/climada_python/pull/911)
+
+### Fixed
+
+- Avoid an issue where a Hazard subselection would have a fraction matrix with only zeros as entries by throwing an error [#866](https://github.com/CLIMADA-project/climada_python/pull/866)
+- Allow downgrading the Python bugfix version to improve environment compatibility [#900](https://github.com/CLIMADA-project/climada_python/pull/900)
+- Fix broken links in `CONTRIBUTING.md` [#900](https://github.com/CLIMADA-project/climada_python/pull/900)
+- When writing `TCTracks` to NetCDF, only apply compression to `float` or `int` data types. This fixes a downstream issue, see [climada_petals#135](https://github.com/CLIMADA-project/climada_petals/issues/135) [#911](https://github.com/CLIMADA-project/climada_python/pull/911)
+
+### Added
+
+- Method `Hazard.check_matrices` for bringing the stored CSR matrices into "canonical format" [#893](https://github.com/CLIMADA-project/climada_python/pull/893)
+- Generic s-shaped impact function via `ImpactFunc.from_poly_s_shape` [#878](https://github.com/CLIMADA-project/climada_python/pull/878)
+- climada.hazard.centroids.centr.Centroids.get_area_pixel
+- climada.hazard.centroids.centr.Centroids.get_dist_coast
+- climada.hazard.centroids.centr.Centroids.get_elevation
+- climada.hazard.centroids.centr.Centroids.get_meta
+- climada.hazard.centroids.centr.Centroids.get_pixel_shapes
+- climada.hazard.centroids.centr.Centroids.to_crs
+- climada.hazard.centroids.centr.Centroids.to_default_crs
+- climada.hazard.centroids.centr.Centroids.write_csv
+- climada.hazard.centroids.centr.Centroids.write_excel
+
+### Deprecated
+
+- climada.hazard.centroids.centr.Centroids.from_lat_lon
+- climada.hazard.centroids.centr.Centroids.def set_area_pixel
+- climada.hazard.centroids.centr.Centroids.def set_area_approx
+- climada.hazard.centroids.centr.Centroids.set_dist_coast
+- climada.hazard.centroids.centr.Centroids.empty_geometry_points
+- climada.hazard.centroids.centr.Centroids.set_meta_to_lat_lon
+- climada.hazard.centroids.centr.Centroids.set_lat_lon_to_meta
+- `scheduler` parameter in `climada.util.coordinates.set_df_geometry_points`, as dask is not used anymore, leaving all calculation to shapely [#912](https://github.com/CLIMADA-project/climada_python/pull/912)
+
+### Removed
+
+- climada.hazard.base.Hazard.clear
+- climada.hazard.base.Hazard.raster_to_vector
+- climada.hazard.base.Hazard.read_mat
+- climada.hazard.base.Hazard.reproject_raster
+- climada.hazard.base.Hazard.set_vector
+- climada.hazard.base.Hazard.vector_to_raster
+- climada.hazard.centroids.centr.Centroids.calc_pixels_polygons
+- climada.hazard.centroids.centr.Centroids.check
+- climada.hazard.centroids.centr.Centroids.clear
+- climada.hazard.centroids.centr.Centroids.equal
+- climada.hazard.centroids.centr.Centroids.from_mat
+- climada.hazard.centroids.centr.Centroids.from_base_grid
+- climada.hazard.centroids.centr.Centroids.read_excel
+- climada.hazard.centroids.centr.Centroids.read_hdf5
+- climada.hazard.centroids.centr.Centroids.read_mat
+- climada.hazard.centroids.centr.Centroids.set_elevation
+- climada.hazard.centroids.centr.Centroids.set_geometry_points
+- climada.hazard.centroids.centr.Centroids.set_lat_lon
+- climada.hazard.centroids.centr.Centroids.set_raster_file
+- climada.hazard.centroids.centr.Centroids.set_raster_from_pnt_bounds
+- climada.hazard.centroids.centr.Centroids.set_vector_file
+- climada.hazard.centroids.centr.Centroids.values_from_raster_files
+- climada.hazard.centroids.centr.Centroids.values_from_vector_files
+- climada.hazard.centroids.centr.generate_nat_earth_centroids
+- `requirements/env_docs.yml`. The regular environment specs are now used to build the online documentation [#687](https://github.com/CLIMADA-project/climada_python/pull/687)
+
 ## 4.1.1
 
 Release date: 2024-02-21
@@ -81,6 +194,7 @@ Changed:
 
 - `geopandas` >=0.13 &rarr; >=0.14
 - `pandas` >=1.5,<2.0 &rarr; >=2.1
+- `salib` >=1.3.0 &rarr; >=1.4.7
 
 Removed:
 

CONTRIBUTING.md

@@ -22,7 +22,7 @@ Please contact the [lead developers](https://wcr.ethz.ch/research/climada.html)
 
 ## Minimal Steps to Contribute
 
-Before you start, please have a look at our [Developer Guide][devguide].
+Before you start, please have a look at our Developer Guide section in the [CLIMADA Docs][docs].
 
 To contribute follow these steps:
 
@@ -65,21 +65,22 @@ To contribute follow these steps:
 
 ## Resources
 
-The CLIMADA documentation provides a [Developer Guide][devguide].
+The [CLIMADA documentation][docs] provides several Developer Guides.
 Here's a selection of the commonly required information:
 
 * How to use Git and GitHub for CLIMADA development: [Development and Git and CLIMADA](https://climada-python.readthedocs.io/en/latest/guide/Guide_Git_Development.html)
-* Coding instructions for CLIMADA: [Python Dos and Don'ts](https://climada-python.readthedocs.io/en/latest/guide/Guide_PythonDos-n-Donts.html), [Performance Tips](https://climada-python.readthedocs.io/en/latest/guide/Guide_Py_Performance.html), [CLIMADA Conventions](https://climada-python.readthedocs.io/en/latest/guide/Guide_Miscellaneous.html)
-* How to execute tests in CLIMADA: [Testing and Continuous Integration][testing]
+* Coding instructions for CLIMADA: [Python Dos and Don'ts](https://climada-python.readthedocs.io/en/latest/guide/Guide_PythonDos-n-Donts.html), [Performance Tips](https://climada-python.readthedocs.io/en/latest/guide/Guide_Py_Performance.html), [CLIMADA Conventions](https://climada-python.readthedocs.io/en/latest/guide/Guide_CLIMADA_conventions.html)
+* How to execute tests in CLIMADA: [Testing][testing] and [Continuous Integration](https://climada-python.readthedocs.io/en/latest/guide/Guide_continuous_integration_GitHub_actions.html)
 
 ## Pull Requests
 
 After developing a new feature, fixing a bug, or updating the tutorials, you can create a [pull request](https://docs.github.com/en/pull-requests) to have your changes reviewed and then merged into the CLIMADA code base.
 To ensure that your pull request can be reviewed quickly and easily, please have a look at the _Resources_ above before opening a pull request.
-In particular, please check out the [Pull Request instructions](https://climada-python.readthedocs.io/en/latest/guide/Guide_Git_Development.html#Pull-requests).
+In particular, please check out the [Pull Request instructions](https://climada-python.readthedocs.io/en/latest/guide/Guide_Git_Development.html#pull-requests).
 
 We provide a description template for pull requests that helps you provide the essential information for reviewers.
 It also contains a checklist for both pull request authors and reviewers to guide the review process.
 
+[docs]: https://climada-python.readthedocs.io/en/latest/
 [devguide]: https://climada-python.readthedocs.io/en/latest/#developer-guide
-[testing]: https://climada-python.readthedocs.io/en/latest/guide/Guide_Continuous_Integration_and_Testing.html
+[testing]: https://climada-python.readthedocs.io/en/latest/guide/Guide_Testing.html

Makefile

@@ -5,8 +5,8 @@
 PYTEST_JUNIT_ARGS = --junitxml=tests_xml/tests.xml
 
 PYTEST_COV_ARGS = \
-	--cov --cov-config=.coveragerc --cov-report html --cov-report xml \
-	--cov-report term:skip-covered
+	--cov --cov-config=.coveragerc --cov-report html:coverage \
+	--cov-report xml:coverage.xml --cov-report term:skip-covered
 
 PYTEST_ARGS = $(PYTEST_JUNIT_ARGS) $(PYTEST_COV_ARGS)
 

climada/_version.py

@@ -1 +1 @@
-__version__ = '4.1.1'
+__version__ = '4.1.2-dev'

climada/conf/climada.conf

@@ -67,7 +67,7 @@
         "cache_db": "{local_data.system}/.downloads.db",
         "cache_enabled": true,
         "cache_dir": "{local_data.system}/.apicache",
-        "supported_hazard_types": ["river_flood", "tropical_cyclone", "storm_europe", "relative_cropyield", "wildfire", "earthquake", "flood"],
-        "supported_exposures_types": ["litpop", "crop_production", "ssp_population"]
+        "supported_hazard_types": ["river_flood", "tropical_cyclone", "storm_europe", "relative_cropyield", "wildfire", "earthquake", "flood", "hail", "aqueduct_coastal_flood"],
+        "supported_exposures_types": ["litpop", "crop_production", "ssp_population", "crops"]
     }
 }

climada/engine/forecast.py

@@ -31,6 +31,7 @@
 from matplotlib.ticker import PercentFormatter, ScalarFormatter
 from matplotlib.colors import ListedColormap, BoundaryNorm
 import cartopy.crs as ccrs
+from matplotlib import colormaps as cm
 import pyproj
 import shapely
 from cartopy.io import shapereader
@@ -88,7 +89,7 @@
 warnprob_colors_extended = np.repeat(warnprob_colors, 10, axis=0)
 CMAP_WARNPROB = ListedColormap(warnprob_colors_extended)
 # colors for impact forecast
-color_map_pre = plt.get_cmap("plasma", 90)
+color_map_pre = cm.get_cmap("plasma").resampled(90)
 impact_colors = color_map_pre(np.linspace(0, 1, 90))
 white_extended = np.repeat([[255 / 255, 255 / 255, 255 / 255, 1]], 10, axis=0)
 impact_colors_extended = np.append(white_extended, impact_colors, axis=0)

climada/engine/impact.py

@@ -937,11 +937,6 @@ def write_hdf5(self, file_path: Union[str, Path], dense_imp_mat: bool=False):
 
         The impact matrix can be stored in a sparse or dense format.
 
-        Notes
-        -----
-        This writer does not support attributes with variable types. Please make sure
-        that ``event_name`` is a list of equally-typed values, e.g., all ``str``.
-
         Parameters
         ----------
         file_path : str or Path
@@ -950,6 +945,11 @@ def write_hdf5(self, file_path: Union[str, Path], dense_imp_mat: bool=False):
             If ``True``, write the impact matrix as dense matrix that can be more easily
             interpreted by common H5 file readers but takes up (vastly) more space.
             Defaults to ``False``.
+
+        Raises
+        ------
+        TypeError
+            If :py:attr:`event_name` does not contain strings exclusively.
         """
         # Define writers for all types (will be filled later)
         type_writers = dict()
@@ -983,7 +983,7 @@ def write(group: h5py.Group, name: str, value: Any):
 
         def _str_type_helper(values: Collection):
             """Return string datatype if we assume 'values' contains strings"""
-            if isinstance(next(iter(values)), str):
+            if all((isinstance(val, str) for val in values)):
                 return h5py.string_dtype()
             return None
 
@@ -1037,6 +1037,8 @@ def write_csr(group, name, value):
             # Now write all attributes
             # NOTE: Remove leading underscore to write '_tot_value' as regular attribute
             for name, value in self.__dict__.items():
+                if name == "event_name" and _str_type_helper(value) is None:
+                    raise TypeError("'event_name' must be a list of strings")
                 write(file, name.lstrip("_"), value)
 
     def write_sparse_csr(self, file_name):
@@ -1240,10 +1242,18 @@ def from_hdf5(cls, file_path: Union[str, Path]):
             ).intersection(file.keys())
             kwargs.update({attr: file[attr][:] for attr in array_attrs})
 
-            # Special handling for 'event_name' because it's a list of strings
+            # Special handling for 'event_name' because it should be a list of strings
             if "event_name" in file:
                 # pylint: disable=no-member
-                kwargs["event_name"] = list(file["event_name"].asstr()[:])
+                try:
+                    event_name = file["event_name"].asstr()[:]
+                except TypeError:
+                    LOGGER.warning(
+                        "'event_name' is not stored as strings. Trying to decode "
+                        "values with 'str()' instead."
+                    )
+                    event_name = map(str, file["event_name"][:])
+                kwargs["event_name"] = list(event_name)
 
         # Create the impact object
         return cls(**kwargs)