From e130f3dd8ac55c745341338d354e135c42a960f1 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Wed, 20 Jul 2022 17:59:42 +0200 Subject: [PATCH 01/16] cosmetics --- climada/engine/impact_calc.py | 15 ++++++++++----- climada/engine/unsequa/unc_output.py | 2 +- climada/entity/exposures/base.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 78c65b3e30..f9fcc503ba 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -61,7 +61,8 @@ def __init__(self, self.exposures = exposures self.impfset = impfset self.hazard = hazard - self._orig_exp_idx = np.arange(self.exposures.gdf.shape[0]) #exposures index to use for matrix reconstruction + # exposures index to use for matrix reconstruction + self._orig_exp_idx = np.arange(self.exposures.gdf.shape[0]) @property def n_exp_pnt(self): @@ -260,7 +261,8 @@ def minimal_exp_gdf(self, impf_col): ) if exp_gdf.size == 0: LOGGER.warning("No exposures with value >0 in the vicinity of the hazard.") - self._orig_exp_idx = mask.nonzero()[0] #update index of kept exposures points in exp_gdf within the full exposures + self._orig_exp_idx = mask.nonzero()[0] # update index of kept exposures points in exp_gdf + # within the full exposures return exp_gdf def imp_mat_gen(self, exp_gdf, impf_col): @@ -381,10 +383,11 @@ def impact_matrix(self, exp_values, cent_idx, impf): scipy.sparse.csr_matrix Impact per event (rows) per exposure point (columns) """ - n_exp_pnt = len(cent_idx) #implicitly checks in matrix assignement whether len(cent_idx) == len(exp_values) + n_exp_pnt = len(cent_idx) # implicitly checks in matrix assignement whether + # len(cent_idx) == len(exp_values) mdr = self.hazard.get_mdr(cent_idx, impf) fract = self.hazard.get_fraction(cent_idx) - exp_values_csr = sparse.csr_matrix( #vector 1 x exp_size + exp_values_csr = sparse.csr_matrix( # vector 1 x exp_size (exp_values, np.arange(n_exp_pnt), [0, n_exp_pnt]), shape=(1, n_exp_pnt)) return fract.multiply(mdr).multiply(exp_values_csr) @@ -393,7 +396,9 @@ def stitch_impact_matrix(self, imp_mat_gen): """ Make an impact matrix from an impact sub-matrix generator """ - data, row, col = np.hstack([ #rows=events index, cols=exposure point index within self.exposures + # rows: events index + # cols: exposure point index within self.exposures + data, row, col = np.hstack([ (mat.data, mat.nonzero()[0], self._orig_exp_idx[idx][mat.nonzero()[1]]) for mat, idx in imp_mat_gen ]) diff --git a/climada/engine/unsequa/unc_output.py b/climada/engine/unsequa/unc_output.py index 0d70d37a41..41c2e566a5 100644 --- a/climada/engine/unsequa/unc_output.py +++ b/climada/engine/unsequa/unc_output.py @@ -962,7 +962,7 @@ def plot_sensitivity_map(self, salib_si='S1', **kwargs): eai_max_si_df = self.get_largest_si(salib_si, metric_list=['eai_exp']) plot_val = eai_max_si_df['param'] - coord = np.array([self.coord_df.latitude, self.coord_df.longitude]).transpose() + coord = np.array([self.coord_df.latitude, self.coord_df.longitude]).transpose() # pylint: disable=no-member if 'var_name' not in kwargs: kwargs['var_name'] = 'Input parameter with largest ' + salib_si if 'title' not in kwargs: diff --git a/climada/entity/exposures/base.py b/climada/entity/exposures/base.py index 309bc4492d..1c0de380fd 100644 --- a/climada/entity/exposures/base.py +++ b/climada/entity/exposures/base.py @@ -417,7 +417,7 @@ def assign_centroids(self, hazard, distance='euclidean', if overwrite: LOGGER.info('Existing centroids will be overwritten for %s', haz_type) else: - return None + return LOGGER.info('Matching %s exposures with %s centroids.', str(self.gdf.shape[0]), str(hazard.centroids.size)) From 542aabfc5521b7382b9b316ade4eb94643594288 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Wed, 20 Jul 2022 18:05:39 +0200 Subject: [PATCH 02/16] impact_calc.ImpactCalc.minimal_exp_gdf: change default to assign_centroids(... overwrite=True) --- climada/engine/impact_calc.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index f9fcc503ba..1dc98ec6b0 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -239,18 +239,23 @@ def _return_empty(self, save_mat): return Impact.from_eih(self.exposures, self.impfset, self.hazard, at_event, eai_exp, aai_agg, imp_mat) - def minimal_exp_gdf(self, impf_col): + def minimal_exp_gdf(self, impf_col, reassign=True): """Get minimal exposures geodataframe for impact computation Parameters ---------- exposures : climada.entity.Exposures hazard : climada.Hazard - impf_col: str + impf_col : str name of the impact function column in exposures.gdf - - """ - self.exposures.assign_centroids(self.hazard, overwrite=False) + reassign : bool, optional + indicates whether centroids are re-assigned to the self.exposures object + or kept from previous impact calculation with a hazard of the same hazard type. + Centroids assignment is an expensive operation, set this to true if you know that + the centroids are the same between two impact calculations. + Default: True, i.e., re-assign + """ + self.exposures.assign_centroids(self.hazard, overwrite=reassign) mask = ( (self.exposures.gdf.value.values != 0) & (self.exposures.gdf[self.hazard.centr_exp_col].values >= 0) From d7834e2b23794073ac6339c91851abfed7fe6c0b Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Thu, 4 Aug 2022 15:20:24 +0200 Subject: [PATCH 03/16] impact_calc: introduce optonal reassign_centroids argument set it to False wherever the assignment is obviously obsolete --- climada/engine/forecast.py | 2 +- climada/engine/impact.py | 8 +++--- climada/engine/impact_calc.py | 34 ++++++++++++++++-------- climada/engine/test/test_cost_benefit.py | 2 +- climada/engine/test/test_impact.py | 6 ++--- climada/engine/test/test_impact_calc.py | 6 ++--- climada/test/test_calibration.py | 3 ++- climada/util/lines_polys_handler.py | 4 +-- 8 files changed, 39 insertions(+), 26 deletions(-) diff --git a/climada/engine/forecast.py b/climada/engine/forecast.py index 860f70a235..cce5bda699 100644 --- a/climada/engine/forecast.py +++ b/climada/engine/forecast.py @@ -297,7 +297,7 @@ def calc(self, force_reassign=False): if force_reassign: self.exposure.assign_centroids(haz_i) self._impact[ind_i].calc( - self.exposure, self.vulnerability, haz_i, save_mat=True + self.exposure, self.vulnerability, haz_i, save_mat=True, reassign_centroids=False ) def plot_imp_map( diff --git a/climada/engine/impact.py b/climada/engine/impact.py index d70cb2d476..a044a0364c 100755 --- a/climada/engine/impact.py +++ b/climada/engine/impact.py @@ -184,7 +184,7 @@ def __init__(self, - def calc(self, exposures, impact_funcs, hazard, save_mat=False): + def calc(self, exposures, impact_funcs, hazard, save_mat=False, reassign_centroids=True): """This function is deprecated, use ImpactCalc.impact and ImpactCalc.insured_impact instead. """ @@ -198,11 +198,11 @@ def calc(self, exposures, impact_funcs, hazard, save_mat=False): " for insured impacts instead. For non-insured impacts " "please use ImpactCalc().impact()" ) - self.__dict__ = impcalc.insured_impact(save_mat).__dict__ + self.__dict__ = impcalc.insured_impact(save_mat, reassign_centroids).__dict__ else: LOGGER.warning("The use of Impact().calc() is deprecated. " "Use ImpactCalc().impact() instead.") - self.__dict__ = impcalc.impact(save_mat).__dict__ + self.__dict__ = impcalc.impact(save_mat, reassign_centroids).__dict__ #TODO: new name @classmethod @@ -1029,7 +1029,7 @@ def video_direct_impact(exp, impf_set, haz_list, file_name='', imp_arr = np.zeros(len(exp.gdf)) for i_time, _ in enumerate(haz_list): imp_tmp = Impact() - imp_tmp.calc(exp, impf_set, haz_list[i_time]) + imp_tmp.calc(exp, impf_set, haz_list[i_time], reassign_centroids=False) imp_arr = np.maximum(imp_arr, imp_tmp.eai_exp) # remove not impacted exposures save_exp = imp_arr > imp_thresh diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 1dc98ec6b0..9bbe6ea9e7 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -102,13 +102,20 @@ def cover(self): if 'cover' in self.exposures.gdf.columns: return self.exposures.gdf['cover'].to_numpy() - def impact(self, save_mat=True): + def impact(self, save_mat=True, reassign_centroids=True): """Compute the impact of a hazard on exposures. Parameters ---------- - save_mat : bool + save_mat : bool, optional if true, save the total impact matrix (events x exposures) + Default: True + reassign_centroids : bool, optional + indicates whether centroids are re-assigned to the self.exposures object + or kept from previous impact calculation with a hazard of the same hazard type. + Centroids assignment is an expensive operation, set this to true if you know that + the centroids are the same between two impact calculations. + Default: True Examples -------- @@ -127,7 +134,7 @@ def impact(self, save_mat=True): the column is added to the exposures geodataframe. """ impf_col = self.exposures.get_impf_column(self.hazard.haz_type) - exp_gdf = self.minimal_exp_gdf(impf_col) + exp_gdf = self.minimal_exp_gdf(impf_col, reassign_centroids) if exp_gdf.size == 0: return self._return_empty(save_mat) LOGGER.info('Calculating impact for %s assets (>0) and %s events.', @@ -137,7 +144,7 @@ def impact(self, save_mat=True): #TODO: make a better impact matrix generator for insured impacts when # the impact matrix is already present - def insured_impact(self, save_mat=False): + def insured_impact(self, save_mat=False, reassign_centroids=True): """Compute the impact of a hazard on exposures with a deductible and/or cover. @@ -148,6 +155,12 @@ def insured_impact(self, save_mat=False): ---------- save_mat : bool if true, save the total impact matrix (events x exposures) + reassign_centroids : bool, optional + indicates whether centroids are re-assigned to the self.exposures object + or kept from previous impact calculation with a hazard of the same hazard type. + Centroids assignment is an expensive operation, set this to true if you know that + the centroids are the same between two impact calculations. + Default: True Examples -------- @@ -170,7 +183,7 @@ def insured_impact(self, save_mat=False): "Please set exposures.gdf.cover" "and/or exposures.gdf.deductible") impf_col = self.exposures.get_impf_column(self.hazard.haz_type) - exp_gdf = self.minimal_exp_gdf(impf_col) + exp_gdf = self.minimal_exp_gdf(impf_col, reassign_centroids) if exp_gdf.size == 0: return self._return_empty(save_mat) LOGGER.info('Calculating impact for %s assets (>0) and %s events.', @@ -239,7 +252,7 @@ def _return_empty(self, save_mat): return Impact.from_eih(self.exposures, self.impfset, self.hazard, at_event, eai_exp, aai_agg, imp_mat) - def minimal_exp_gdf(self, impf_col, reassign=True): + def minimal_exp_gdf(self, impf_col, reassign_centroids): """Get minimal exposures geodataframe for impact computation Parameters @@ -247,15 +260,14 @@ def minimal_exp_gdf(self, impf_col, reassign=True): exposures : climada.entity.Exposures hazard : climada.Hazard impf_col : str - name of the impact function column in exposures.gdf - reassign : bool, optional - indicates whether centroids are re-assigned to the self.exposures object + Name of the impact function column in exposures.gdf + reassign_centroids : bool + Indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. Centroids assignment is an expensive operation, set this to true if you know that the centroids are the same between two impact calculations. - Default: True, i.e., re-assign """ - self.exposures.assign_centroids(self.hazard, overwrite=reassign) + self.exposures.assign_centroids(self.hazard, overwrite=reassign_centroids) mask = ( (self.exposures.gdf.value.values != 0) & (self.exposures.gdf[self.hazard.centr_exp_col].values >= 0) diff --git a/climada/engine/test/test_cost_benefit.py b/climada/engine/test/test_cost_benefit.py index 33cb7da845..9d87e6c4c7 100644 --- a/climada/engine/test/test_cost_benefit.py +++ b/climada/engine/test/test_cost_benefit.py @@ -820,7 +820,7 @@ def test_impact(self): hazard = Hazard.from_mat(HAZ_TEST_MAT) impact = Impact() ent.exposures.assign_centroids(hazard) - impact.calc(ent.exposures, ent.impact_funcs, hazard) + impact.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) return impact def test_risk_aai_agg_pass(self): diff --git a/climada/engine/test/test_impact.py b/climada/engine/test/test_impact.py index 5a6a09c065..12bc4bc497 100644 --- a/climada/engine/test/test_impact.py +++ b/climada/engine/test/test_impact.py @@ -298,7 +298,7 @@ def test_excel_io(self): imp_write = Impact() ent.exposures.assign_centroids(hazard) - imp_write.calc(ent.exposures, ent.impact_funcs, hazard) + imp_write.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) file_name = DATA_FOLDER.joinpath('test.xlsx') imp_write.write_excel(file_name) @@ -351,7 +351,7 @@ def test_local_exceedance_imp_pass(self): # Assign centroids to exposures ent.exposures.assign_centroids(hazard) # Compute the impact over the whole exposures - impact.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True) + impact.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, reassign_centroids=False) # Compute the impact per return period over the whole exposures impact_rp = impact.local_exceedance_imp(return_periods=(10, 40)) @@ -561,7 +561,7 @@ def test_select_event_identity_pass(self): ent.exposures.assign_centroids(hazard) # Compute the impact over the whole exposures - imp.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True) + imp.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, reassign_centroids=False) sel_imp = imp.select(event_ids=imp.event_id, event_names=imp.event_name, diff --git a/climada/engine/test/test_impact_calc.py b/climada/engine/test/test_impact_calc.py index d426157a6a..eda69cb281 100644 --- a/climada/engine/test/test_impact_calc.py +++ b/climada/engine/test/test_impact_calc.py @@ -133,7 +133,7 @@ def test_calc_impact_TC_pass(self): HAZf = deepcopy(HAZ) HAZf.fraction *= 0.6 icalc = ImpactCalc(ENT.exposures, ENT.impact_funcs, HAZf) - impact = icalc.impact() + impact = icalc.impact(reassign_centroids=False) self.assertEqual(icalc.n_events, len(impact.at_event)) self.assertEqual(0, impact.at_event[0]) self.assertEqual(0, impact.at_event[7225]) @@ -191,7 +191,7 @@ def test_empty_impact(self): at_event = np.zeros(HAZ.size) check_impact(self, impact, HAZ, exp, aai_agg, eai_exp, at_event, None) - impact = icalc.impact(save_mat=True) + impact = icalc.impact(save_mat=True, reassign_centroids=False) imp_mat_array = sparse.csr_matrix((HAZ.size, len(exp.gdf))).toarray() check_impact(self, impact, HAZ, exp, aai_agg, eai_exp, at_event, imp_mat_array) @@ -204,7 +204,7 @@ def test_single_event_impact(self): eai_exp = np.zeros(len(ENT.exposures.gdf)) at_event = np.array([0]) check_impact(self, impact, haz, ENT.exposures, aai_agg, eai_exp, at_event, None) - impact = icalc.impact(save_mat=True) + impact = icalc.impact(save_mat=True, reassign_centroids=False) imp_mat_array = sparse.csr_matrix((haz.size, len(ENT.exposures.gdf))).toarray() check_impact(self, impact, haz, ENT.exposures, aai_agg, eai_exp, at_event, imp_mat_array) diff --git a/climada/test/test_calibration.py b/climada/test/test_calibration.py index 54d3a28a3c..9b25865f8c 100644 --- a/climada/test/test_calibration.py +++ b/climada/test/test_calibration.py @@ -18,6 +18,7 @@ Test Calibration class. """ +import re import unittest from pathlib import Path import pandas as pd @@ -69,7 +70,7 @@ def test_calib_instance(self): yearly_impact=True) # calc Impact as comparison impact = Impact() - impact.calc(ent.exposures, ent.impact_funcs, hazard) + impact.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) IYS = impact.calc_impact_year_set(all_years=True) # do the tests diff --git a/climada/util/lines_polys_handler.py b/climada/util/lines_polys_handler.py index 0a7c714004..2227b83f4f 100755 --- a/climada/util/lines_polys_handler.py +++ b/climada/util/lines_polys_handler.py @@ -124,7 +124,7 @@ def calc_geom_impact( # compute point impact impact_pnt = Impact() - impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True) + impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, reassign_centroids=False) # re-aggregate impact to original exposure geometry impact_agg = impact_pnt_agg(impact_pnt, exp_pnt.gdf, agg_met) @@ -295,7 +295,7 @@ def calc_grid_impact( # compute point impact impact_pnt = Impact() - impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True) + impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, reassign_centroids=False) # re-aggregate impact to original exposure geometry impact_agg = impact_pnt_agg(impact_pnt, exp_pnt.gdf, agg_met) From 50e6469808e3ce72acf75976c0c606251bf57f40 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Thu, 4 Aug 2022 16:10:14 +0200 Subject: [PATCH 04/16] test_impact_calc: fix failing tests (due to reassignment) --- climada/engine/test/test_impact_calc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/climada/engine/test/test_impact_calc.py b/climada/engine/test/test_impact_calc.py index eda69cb281..7d08a831a0 100644 --- a/climada/engine/test/test_impact_calc.py +++ b/climada/engine/test/test_impact_calc.py @@ -152,7 +152,7 @@ def test_calc_impact_RF_pass(self): exp = Exposures.from_hdf5(get_test_file('test_exposure_US_flood_random_locations')) impf_set = ImpactFuncSet.from_excel(Path(__file__).parent / 'data' / 'flood_imp_func_set.xls') icalc = ImpactCalc(exp, impf_set, haz) - impact = icalc.impact() + impact = icalc.impact(reassign_centroids=False) aai_agg = 161436.05112960344 eai_exp = np.array([ 1.61159701e+05, 1.33742847e+02, 0.00000000e+00, 4.21352988e-01, @@ -181,11 +181,11 @@ def test_calc_impact_RF_pass(self): check_impact(self, impact, haz, exp, aai_agg, eai_exp, at_event, imp_mat_array) def test_empty_impact(self): - """Check that empty impact is returned if no centroids matching the exposures""" + """Check that empty impact is returned if no centroids match the exposures""" exp = ENT.exposures.copy() exp.gdf['centr_TC'] = -1 icalc = ImpactCalc(exp, ENT.impact_funcs, HAZ) - impact = icalc.impact() + impact = icalc.impact(reassign_centroids=False) aai_agg = 0.0 eai_exp = np.zeros(len(exp.gdf)) at_event = np.zeros(HAZ.size) @@ -299,7 +299,7 @@ def test_calc_insured_impact_fail(self): def test_minimal_exp_gdf(self): """Test obtain minimal exposures gdf""" icalc = ImpactCalc(ENT.exposures, ENT.impact_funcs, HAZ) - exp_min_gdf = icalc.minimal_exp_gdf('impf_TC') + exp_min_gdf = icalc.minimal_exp_gdf('impf_TC', False) self.assertSetEqual( set(exp_min_gdf.columns), set(['value', 'impf_TC', 'centr_TC']) ) From bfc94ae16838cc9a098484071dce1b0f0f4f9cc2 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 12:54:26 +0200 Subject: [PATCH 05/16] forecast.Forecast.calc: simply pass the force_reassign parameter to the Impact.calc method --- climada/engine/forecast.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/climada/engine/forecast.py b/climada/engine/forecast.py index cce5bda699..b51cee451a 100644 --- a/climada/engine/forecast.py +++ b/climada/engine/forecast.py @@ -293,11 +293,9 @@ def calc(self, force_reassign=False): """ # calc impact for ind_i, haz_i in enumerate(self.hazard): - # force reassign - if force_reassign: - self.exposure.assign_centroids(haz_i) self._impact[ind_i].calc( - self.exposure, self.vulnerability, haz_i, save_mat=True, reassign_centroids=False + self.exposure, self.vulnerability, haz_i, + save_mat=True, reassign_centroids=force_reassign ) def plot_imp_map( From 5df35b74a858dca7699690dc93107aedc14876ce Mon Sep 17 00:00:00 2001 From: Emanuel Schmid <51439563+emanuel-schmid@users.noreply.github.com> Date: Fri, 5 Aug 2022 13:01:55 +0200 Subject: [PATCH 06/16] Apply suggestions from code review Co-authored-by: Lukas Riedel <34276446+peanutfun@users.noreply.github.com> --- climada/engine/impact_calc.py | 15 +++++++++------ climada/test/test_calibration.py | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 9bbe6ea9e7..c44f7f67f1 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -113,8 +113,9 @@ def impact(self, save_mat=True, reassign_centroids=True): reassign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation, set this to true if you know that - the centroids are the same between two impact calculations. + Centroids assignment is an expensive operation; set this to ``False`` to save + computation time if the centroids have not changed since the last impact + calculation. Default: True Examples @@ -158,8 +159,9 @@ def insured_impact(self, save_mat=False, reassign_centroids=True): reassign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation, set this to true if you know that - the centroids are the same between two impact calculations. + Centroids assignment is an expensive operation; set this to ``False`` to save + computation time if the centroids have not changed since the last impact + calculation. Default: True Examples @@ -264,8 +266,9 @@ def minimal_exp_gdf(self, impf_col, reassign_centroids): reassign_centroids : bool Indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation, set this to true if you know that - the centroids are the same between two impact calculations. + Centroids assignment is an expensive operation; set this to ``False`` to save + computation time if the centroids have not changed since the last impact + calculation. """ self.exposures.assign_centroids(self.hazard, overwrite=reassign_centroids) mask = ( diff --git a/climada/test/test_calibration.py b/climada/test/test_calibration.py index 9b25865f8c..abe1604e57 100644 --- a/climada/test/test_calibration.py +++ b/climada/test/test_calibration.py @@ -18,7 +18,6 @@ Test Calibration class. """ -import re import unittest from pathlib import Path import pandas as pd From bad251376a1127537ad71327b730a25b3e129c79 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 13:36:28 +0200 Subject: [PATCH 07/16] rename `reassign_centroids` arguments to `assign_centroids` consequently, don't assign if not set to true --- climada/engine/forecast.py | 2 +- climada/engine/impact.py | 10 ++++++---- climada/engine/impact_calc.py | 24 +++++++++++++++--------- climada/engine/test/test_cost_benefit.py | 2 +- climada/engine/test/test_impact.py | 6 +++--- climada/engine/test/test_impact_calc.py | 10 +++++----- climada/test/test_calibration.py | 2 +- climada/util/lines_polys_handler.py | 4 ++-- 8 files changed, 34 insertions(+), 26 deletions(-) diff --git a/climada/engine/forecast.py b/climada/engine/forecast.py index b51cee451a..e32a290c48 100644 --- a/climada/engine/forecast.py +++ b/climada/engine/forecast.py @@ -295,7 +295,7 @@ def calc(self, force_reassign=False): for ind_i, haz_i in enumerate(self.hazard): self._impact[ind_i].calc( self.exposure, self.vulnerability, haz_i, - save_mat=True, reassign_centroids=force_reassign + save_mat=True, assign_centroids=force_reassign ) def plot_imp_map( diff --git a/climada/engine/impact.py b/climada/engine/impact.py index a044a0364c..517b00eac6 100755 --- a/climada/engine/impact.py +++ b/climada/engine/impact.py @@ -184,7 +184,7 @@ def __init__(self, - def calc(self, exposures, impact_funcs, hazard, save_mat=False, reassign_centroids=True): + def calc(self, exposures, impact_funcs, hazard, save_mat=False, assign_centroids=True): """This function is deprecated, use ImpactCalc.impact and ImpactCalc.insured_impact instead. """ @@ -198,11 +198,11 @@ def calc(self, exposures, impact_funcs, hazard, save_mat=False, reassign_centroi " for insured impacts instead. For non-insured impacts " "please use ImpactCalc().impact()" ) - self.__dict__ = impcalc.insured_impact(save_mat, reassign_centroids).__dict__ + self.__dict__ = impcalc.insured_impact(save_mat, assign_centroids).__dict__ else: LOGGER.warning("The use of Impact().calc() is deprecated. " "Use ImpactCalc().impact() instead.") - self.__dict__ = impcalc.impact(save_mat, reassign_centroids).__dict__ + self.__dict__ = impcalc.impact(save_mat, assign_centroids).__dict__ #TODO: new name @classmethod @@ -1027,9 +1027,11 @@ def video_direct_impact(exp, impf_set, haz_list, file_name='', imp_list = [] exp_list = [] imp_arr = np.zeros(len(exp.gdf)) + # assign centroids once for all + exp.assign_centroids(haz_list[0]) for i_time, _ in enumerate(haz_list): imp_tmp = Impact() - imp_tmp.calc(exp, impf_set, haz_list[i_time], reassign_centroids=False) + imp_tmp.calc(exp, impf_set, haz_list[i_time], assign_centroids=False) imp_arr = np.maximum(imp_arr, imp_tmp.eai_exp) # remove not impacted exposures save_exp = imp_arr > imp_thresh diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index c44f7f67f1..9c9c0f8e94 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -102,7 +102,7 @@ def cover(self): if 'cover' in self.exposures.gdf.columns: return self.exposures.gdf['cover'].to_numpy() - def impact(self, save_mat=True, reassign_centroids=True): + def impact(self, save_mat=True, assign_centroids=True): """Compute the impact of a hazard on exposures. Parameters @@ -110,7 +110,7 @@ def impact(self, save_mat=True, reassign_centroids=True): save_mat : bool, optional if true, save the total impact matrix (events x exposures) Default: True - reassign_centroids : bool, optional + assign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. Centroids assignment is an expensive operation; set this to ``False`` to save @@ -135,7 +135,7 @@ def impact(self, save_mat=True, reassign_centroids=True): the column is added to the exposures geodataframe. """ impf_col = self.exposures.get_impf_column(self.hazard.haz_type) - exp_gdf = self.minimal_exp_gdf(impf_col, reassign_centroids) + exp_gdf = self.minimal_exp_gdf(impf_col, assign_centroids) if exp_gdf.size == 0: return self._return_empty(save_mat) LOGGER.info('Calculating impact for %s assets (>0) and %s events.', @@ -145,7 +145,7 @@ def impact(self, save_mat=True, reassign_centroids=True): #TODO: make a better impact matrix generator for insured impacts when # the impact matrix is already present - def insured_impact(self, save_mat=False, reassign_centroids=True): + def insured_impact(self, save_mat=False, assign_centroids=True): """Compute the impact of a hazard on exposures with a deductible and/or cover. @@ -156,7 +156,7 @@ def insured_impact(self, save_mat=False, reassign_centroids=True): ---------- save_mat : bool if true, save the total impact matrix (events x exposures) - reassign_centroids : bool, optional + assign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. Centroids assignment is an expensive operation; set this to ``False`` to save @@ -185,7 +185,7 @@ def insured_impact(self, save_mat=False, reassign_centroids=True): "Please set exposures.gdf.cover" "and/or exposures.gdf.deductible") impf_col = self.exposures.get_impf_column(self.hazard.haz_type) - exp_gdf = self.minimal_exp_gdf(impf_col, reassign_centroids) + exp_gdf = self.minimal_exp_gdf(impf_col, assign_centroids) if exp_gdf.size == 0: return self._return_empty(save_mat) LOGGER.info('Calculating impact for %s assets (>0) and %s events.', @@ -254,7 +254,7 @@ def _return_empty(self, save_mat): return Impact.from_eih(self.exposures, self.impfset, self.hazard, at_event, eai_exp, aai_agg, imp_mat) - def minimal_exp_gdf(self, impf_col, reassign_centroids): + def minimal_exp_gdf(self, impf_col, assign_centroids): """Get minimal exposures geodataframe for impact computation Parameters @@ -263,14 +263,20 @@ def minimal_exp_gdf(self, impf_col, reassign_centroids): hazard : climada.Hazard impf_col : str Name of the impact function column in exposures.gdf - reassign_centroids : bool + assign_centroids : bool Indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. Centroids assignment is an expensive operation; set this to ``False`` to save computation time if the centroids have not changed since the last impact calculation. """ - self.exposures.assign_centroids(self.hazard, overwrite=reassign_centroids) + if assign_centroids: + self.exposures.assign_centroids(self.hazard, overwrite=True) + elif self.hazard.centr_exp_col not in self.exposures.gdf.column: + raise ValueError("'assign_centroids' is set to 'False' but no centroids are assigned" + f" for the given hazard type ({self.hazard.tag.haz_type})." + " Run `asssign_centroids` beforehand or set the 'assign_centroids'" + " to 'True'") mask = ( (self.exposures.gdf.value.values != 0) & (self.exposures.gdf[self.hazard.centr_exp_col].values >= 0) diff --git a/climada/engine/test/test_cost_benefit.py b/climada/engine/test/test_cost_benefit.py index 9d87e6c4c7..8b978e78f8 100644 --- a/climada/engine/test/test_cost_benefit.py +++ b/climada/engine/test/test_cost_benefit.py @@ -820,7 +820,7 @@ def test_impact(self): hazard = Hazard.from_mat(HAZ_TEST_MAT) impact = Impact() ent.exposures.assign_centroids(hazard) - impact.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) + impact.calc(ent.exposures, ent.impact_funcs, hazard, assign_centroids=False) return impact def test_risk_aai_agg_pass(self): diff --git a/climada/engine/test/test_impact.py b/climada/engine/test/test_impact.py index 12bc4bc497..054384f9c3 100644 --- a/climada/engine/test/test_impact.py +++ b/climada/engine/test/test_impact.py @@ -298,7 +298,7 @@ def test_excel_io(self): imp_write = Impact() ent.exposures.assign_centroids(hazard) - imp_write.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) + imp_write.calc(ent.exposures, ent.impact_funcs, hazard, assign_centroids=False) file_name = DATA_FOLDER.joinpath('test.xlsx') imp_write.write_excel(file_name) @@ -351,7 +351,7 @@ def test_local_exceedance_imp_pass(self): # Assign centroids to exposures ent.exposures.assign_centroids(hazard) # Compute the impact over the whole exposures - impact.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, reassign_centroids=False) + impact.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, assign_centroids=False) # Compute the impact per return period over the whole exposures impact_rp = impact.local_exceedance_imp(return_periods=(10, 40)) @@ -561,7 +561,7 @@ def test_select_event_identity_pass(self): ent.exposures.assign_centroids(hazard) # Compute the impact over the whole exposures - imp.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, reassign_centroids=False) + imp.calc(ent.exposures, ent.impact_funcs, hazard, save_mat=True, assign_centroids=False) sel_imp = imp.select(event_ids=imp.event_id, event_names=imp.event_name, diff --git a/climada/engine/test/test_impact_calc.py b/climada/engine/test/test_impact_calc.py index 7d08a831a0..4de8e4714c 100644 --- a/climada/engine/test/test_impact_calc.py +++ b/climada/engine/test/test_impact_calc.py @@ -133,7 +133,7 @@ def test_calc_impact_TC_pass(self): HAZf = deepcopy(HAZ) HAZf.fraction *= 0.6 icalc = ImpactCalc(ENT.exposures, ENT.impact_funcs, HAZf) - impact = icalc.impact(reassign_centroids=False) + impact = icalc.impact(assign_centroids=False) self.assertEqual(icalc.n_events, len(impact.at_event)) self.assertEqual(0, impact.at_event[0]) self.assertEqual(0, impact.at_event[7225]) @@ -152,7 +152,7 @@ def test_calc_impact_RF_pass(self): exp = Exposures.from_hdf5(get_test_file('test_exposure_US_flood_random_locations')) impf_set = ImpactFuncSet.from_excel(Path(__file__).parent / 'data' / 'flood_imp_func_set.xls') icalc = ImpactCalc(exp, impf_set, haz) - impact = icalc.impact(reassign_centroids=False) + impact = icalc.impact(assign_centroids=False) aai_agg = 161436.05112960344 eai_exp = np.array([ 1.61159701e+05, 1.33742847e+02, 0.00000000e+00, 4.21352988e-01, @@ -185,13 +185,13 @@ def test_empty_impact(self): exp = ENT.exposures.copy() exp.gdf['centr_TC'] = -1 icalc = ImpactCalc(exp, ENT.impact_funcs, HAZ) - impact = icalc.impact(reassign_centroids=False) + impact = icalc.impact(assign_centroids=False) aai_agg = 0.0 eai_exp = np.zeros(len(exp.gdf)) at_event = np.zeros(HAZ.size) check_impact(self, impact, HAZ, exp, aai_agg, eai_exp, at_event, None) - impact = icalc.impact(save_mat=True, reassign_centroids=False) + impact = icalc.impact(save_mat=True, assign_centroids=False) imp_mat_array = sparse.csr_matrix((HAZ.size, len(exp.gdf))).toarray() check_impact(self, impact, HAZ, exp, aai_agg, eai_exp, at_event, imp_mat_array) @@ -204,7 +204,7 @@ def test_single_event_impact(self): eai_exp = np.zeros(len(ENT.exposures.gdf)) at_event = np.array([0]) check_impact(self, impact, haz, ENT.exposures, aai_agg, eai_exp, at_event, None) - impact = icalc.impact(save_mat=True, reassign_centroids=False) + impact = icalc.impact(save_mat=True, assign_centroids=False) imp_mat_array = sparse.csr_matrix((haz.size, len(ENT.exposures.gdf))).toarray() check_impact(self, impact, haz, ENT.exposures, aai_agg, eai_exp, at_event, imp_mat_array) diff --git a/climada/test/test_calibration.py b/climada/test/test_calibration.py index abe1604e57..9148340139 100644 --- a/climada/test/test_calibration.py +++ b/climada/test/test_calibration.py @@ -69,7 +69,7 @@ def test_calib_instance(self): yearly_impact=True) # calc Impact as comparison impact = Impact() - impact.calc(ent.exposures, ent.impact_funcs, hazard, reassign_centroids=False) + impact.calc(ent.exposures, ent.impact_funcs, hazard, assign_centroids=False) IYS = impact.calc_impact_year_set(all_years=True) # do the tests diff --git a/climada/util/lines_polys_handler.py b/climada/util/lines_polys_handler.py index 2227b83f4f..a466e78900 100755 --- a/climada/util/lines_polys_handler.py +++ b/climada/util/lines_polys_handler.py @@ -124,7 +124,7 @@ def calc_geom_impact( # compute point impact impact_pnt = Impact() - impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, reassign_centroids=False) + impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, assign_centroids=False) # re-aggregate impact to original exposure geometry impact_agg = impact_pnt_agg(impact_pnt, exp_pnt.gdf, agg_met) @@ -295,7 +295,7 @@ def calc_grid_impact( # compute point impact impact_pnt = Impact() - impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, reassign_centroids=False) + impact_pnt.calc(exp_pnt, impf_set, haz, save_mat=True, assign_centroids=False) # re-aggregate impact to original exposure geometry impact_agg = impact_pnt_agg(impact_pnt, exp_pnt.gdf, agg_met) From 9218ba02bb368b2ea921c80f0075f013ab9d0073 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 13:53:12 +0200 Subject: [PATCH 08/16] calibration_opt: skip centroids assignment in calib_instance do it manually before the method is called --- climada/engine/calibration_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/climada/engine/calibration_opt.py b/climada/engine/calibration_opt.py index 491b6dff5b..7426cf350e 100644 --- a/climada/engine/calibration_opt.py +++ b/climada/engine/calibration_opt.py @@ -69,7 +69,7 @@ def calib_instance(hazard, exposure, impact_func, df_out=pd.DataFrame(), IFS = ImpactFuncSet() IFS.append(impact_func) impacts = Impact() - impacts.calc(exposure, IFS, hazard) + impacts.calc(exposure, IFS, hazard, assign_centroids=False) if yearly_impact: # impact per year IYS = impacts.calc_impact_year_set(all_years=True) # Loop over whole year range: @@ -333,6 +333,7 @@ def calib_all(hazard, exposure, impf_name_or_instance, param_full_dict, # prepare hazard and exposure region_ids = list(np.unique(exposure.region_id)) hazard_type = hazard.tag.haz_type + exposure.assign_centroids(hazard) # prepare impact data if isinstance(impact_data_source, pd.DataFrame): df_impact_data = impact_data_source @@ -398,6 +399,7 @@ def calib_optimize(hazard, exposure, impf_name_or_instance, param_dict, # prepare hazard and exposure region_ids = list(np.unique(exposure.region_id)) hazard_type = hazard.tag.haz_type + exposure.assign_centroids(hazard) # prepare impact data if isinstance(impact_data_source, pd.DataFrame): df_impact_data = impact_data_source From 648c0309839b2c44feb83c4941578faca78d213d Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 13:57:20 +0200 Subject: [PATCH 09/16] fix typo --- climada/engine/impact_calc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 9c9c0f8e94..a678030653 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -272,7 +272,7 @@ def minimal_exp_gdf(self, impf_col, assign_centroids): """ if assign_centroids: self.exposures.assign_centroids(self.hazard, overwrite=True) - elif self.hazard.centr_exp_col not in self.exposures.gdf.column: + elif self.hazard.centr_exp_col not in self.exposures.gdf.columns: raise ValueError("'assign_centroids' is set to 'False' but no centroids are assigned" f" for the given hazard type ({self.hazard.tag.haz_type})." " Run `asssign_centroids` beforehand or set the 'assign_centroids'" From caf8dfdf73513b4d86f321e5bbd6362fbc3c81d8 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 15:12:47 +0200 Subject: [PATCH 10/16] PEP8 cosmetics --- climada/engine/calibration_opt.py | 46 +++++++++++++++---------------- climada/engine/impact_calc.py | 6 ++-- climada/entity/measures/base.py | 10 ++++--- climada/hazard/trop_cyclone.py | 4 +-- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/climada/engine/calibration_opt.py b/climada/engine/calibration_opt.py index 7426cf350e..290d862b04 100644 --- a/climada/engine/calibration_opt.py +++ b/climada/engine/calibration_opt.py @@ -66,26 +66,26 @@ def calib_instance(hazard, exposure, impact_func, df_out=pd.DataFrame(), DataFrame with modelled impact written to rows for each year or event. """ - IFS = ImpactFuncSet() - IFS.append(impact_func) + ifs = ImpactFuncSet() + ifs.append(impact_func) impacts = Impact() - impacts.calc(exposure, IFS, hazard, assign_centroids=False) + impacts.calc(exposure, ifs, hazard, assign_centroids=False) if yearly_impact: # impact per year - IYS = impacts.calc_impact_year_set(all_years=True) + iys = impacts.calc_impact_year_set(all_years=True) # Loop over whole year range: if df_out.empty | df_out.index.shape[0] == 1: - for cnt_, year in enumerate(np.sort(list((IYS.keys())))): + for cnt_, year in enumerate(np.sort(list((iys.keys())))): if cnt_ > 0: df_out.loc[cnt_] = df_out.loc[0] # copy info from first row - if year in IYS: - df_out.loc[cnt_, 'impact_CLIMADA'] = IYS[year] + if year in iys: + df_out.loc[cnt_, 'impact_CLIMADA'] = iys[year] else: df_out.loc[cnt_, 'impact_CLIMADA'] = 0.0 df_out.loc[cnt_, 'year'] = year else: - years_in_common = df_out.loc[df_out['year'].isin(np.sort(list((IYS.keys())))), 'year'] + years_in_common = df_out.loc[df_out['year'].isin(np.sort(list((iys.keys())))), 'year'] for cnt_, year in years_in_common.iteritems(): - df_out.loc[df_out['year'] == year, 'impact_CLIMADA'] = IYS[year] + df_out.loc[df_out['year'] == year, 'impact_CLIMADA'] = iys[year] else: # impact per event @@ -138,21 +138,21 @@ def init_impf(impf_name_or_instance, param_dict, df_out=pd.DataFrame(index=[0])) (index=0) defined with values. The impact function parameters from param_dict are represented here. """ - ImpactFunc_final = None + impact_func_final = None if isinstance(impf_name_or_instance, str): if impf_name_or_instance == 'emanuel': - ImpactFunc_final = ImpfTropCyclone.from_emanuel_usa(**param_dict) - ImpactFunc_final.haz_type = 'TC' - ImpactFunc_final.id = 1 + impact_func_final = ImpfTropCyclone.from_emanuel_usa(**param_dict) + impact_func_final.haz_type = 'TC' + impact_func_final.id = 1 df_out['impact_function'] = impf_name_or_instance elif isinstance(impf_name_or_instance, impact_funcs.ImpactFunc): - ImpactFunc_final = change_impf(impf_name_or_instance, param_dict) + impact_func_final = change_impf(impf_name_or_instance, param_dict) df_out['impact_function'] = ('given_' + - ImpactFunc_final.haz_type + - str(ImpactFunc_final.id)) + impact_func_final.haz_type + + str(impact_func_final.id)) for key, val in param_dict.items(): df_out[key] = val - return ImpactFunc_final, df_out + return impact_func_final, df_out def change_impf(impf_instance, param_dict): """apply a shifting or a scaling defined in param_dict to the impact @@ -348,8 +348,8 @@ def calib_all(hazard, exposure, impf_name_or_instance, param_full_dict, for param_dict in params_generator: print(param_dict) df_out = copy.deepcopy(df_impact_data) - ImpactFunc_final, df_out = init_impf(impf_name_or_instance, param_dict, df_out) - df_out = calib_instance(hazard, exposure, ImpactFunc_final, df_out, yearly_impact) + impact_func_final, df_out = init_impf(impf_name_or_instance, param_dict, df_out) + df_out = calib_instance(hazard, exposure, impact_func_final, df_out, yearly_impact) if df_result is None: df_result = copy.deepcopy(df_out) else: @@ -410,8 +410,8 @@ def calib_optimize(hazard, exposure, impf_name_or_instance, param_dict, else: raise ValueError('other impact data sources not yet implemented.') # definie specific function to - def specific_calib(x): - param_dict_temp = dict(zip(param_dict.keys(), x)) + def specific_calib(values): + param_dict_temp = dict(zip(param_dict.keys(), values)) print(param_dict_temp) return calib_instance(hazard, exposure, init_impf(impf_name_or_instance, param_dict_temp)[0], @@ -429,8 +429,8 @@ def specific_calib(x): {'type': 'ineq', 'fun': lambda x: x[1]}] - x0 = list(param_dict.values()) - res = minimize(specific_calib, x0, + values = list(param_dict.values()) + res = minimize(specific_calib, values, # bounds=bounds, # bounds=((0.0, np.inf), (0.0, np.inf), (0.0, 1.0)), constraints=cons, diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index a678030653..4aa28636c7 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -113,7 +113,7 @@ def impact(self, save_mat=True, assign_centroids=True): assign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation; set this to ``False`` to save + Centroids assignment is an expensive operation; set this to ``False`` to save computation time if the centroids have not changed since the last impact calculation. Default: True @@ -159,7 +159,7 @@ def insured_impact(self, save_mat=False, assign_centroids=True): assign_centroids : bool, optional indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation; set this to ``False`` to save + Centroids assignment is an expensive operation; set this to ``False`` to save computation time if the centroids have not changed since the last impact calculation. Default: True @@ -266,7 +266,7 @@ def minimal_exp_gdf(self, impf_col, assign_centroids): assign_centroids : bool Indicates whether centroids are re-assigned to the self.exposures object or kept from previous impact calculation with a hazard of the same hazard type. - Centroids assignment is an expensive operation; set this to ``False`` to save + Centroids assignment is an expensive operation; set this to ``False`` to save computation time if the centroids have not changed since the last impact calculation. """ diff --git a/climada/entity/measures/base.py b/climada/entity/measures/base.py index e3e827aa49..9ebd6f865e 100755 --- a/climada/entity/measures/base.py +++ b/climada/entity/measures/base.py @@ -282,12 +282,14 @@ def _change_exposures_impf(self, exposures): from_id = int(self.imp_fun_map[0:self.imp_fun_map.find('to')]) to_id = int(self.imp_fun_map[self.imp_fun_map.find('to') + 2:]) try: - exp_change = np.argwhere(new_exp.gdf[INDICATOR_IMPF + self.haz_type].values == from_id).\ - reshape(-1) + exp_change = np.argwhere( + new_exp.gdf[INDICATOR_IMPF + self.haz_type].values == from_id + ).reshape(-1) new_exp.gdf[INDICATOR_IMPF + self.haz_type].values[exp_change] = to_id except KeyError: - exp_change = np.argwhere(new_exp.gdf[INDICATOR_IMPF].values == from_id).\ - reshape(-1) + exp_change = np.argwhere( + new_exp.gdf[INDICATOR_IMPF].values == from_id + ).reshape(-1) new_exp.gdf[INDICATOR_IMPF].values[exp_change] = to_id return new_exp diff --git a/climada/hazard/trop_cyclone.py b/climada/hazard/trop_cyclone.py index 5972129ded..f4fc88446e 100644 --- a/climada/hazard/trop_cyclone.py +++ b/climada/hazard/trop_cyclone.py @@ -388,7 +388,7 @@ def run(node): if file_name: LOGGER.info('Generating video %s', file_name) - fig, axis, fontsize = u_plot.make_map(figsize=figsize, adapt_fontsize=adapt_fontsize) + fig, axis, _fontsize = u_plot.make_map(figsize=figsize, adapt_fontsize=adapt_fontsize) pbar = tqdm(total=idx_plt.size - 2) ani = animation.FuncAnimation(fig, run, frames=idx_plt.size - 2, interval=500, blit=False) @@ -873,7 +873,7 @@ def _v_max_s_holland_2008(penv, pcen, b_s): v_squared = b_s / (RHO_AIR * np.exp(1)) * 100 * (penv - pcen) return np.sqrt(v_squared) -def _B_holland_1980(gradient_winds, penv, pcen): +def _B_holland_1980(gradient_winds, penv, pcen): # pylint: disable=invalid-name """Holland's 1980 B-value computation for gradient-level winds. The parameter applies to gradient-level winds (about 1000 metres above the earth's surface). From f0c609d2c42d561feede19c3f60b897cb59ed749 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 15:25:39 +0200 Subject: [PATCH 11/16] forecast.calc: there is no implicit centroid assignment in impact.calc anymore --- climada/engine/forecast.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/climada/engine/forecast.py b/climada/engine/forecast.py index e32a290c48..236831405f 100644 --- a/climada/engine/forecast.py +++ b/climada/engine/forecast.py @@ -292,10 +292,11 @@ def calc(self, force_reassign=False): default is false. """ # calc impact + self.exposure.assign_centroids(self.hazard, overwrite=force_reassign) for ind_i, haz_i in enumerate(self.hazard): self._impact[ind_i].calc( self.exposure, self.vulnerability, haz_i, - save_mat=True, assign_centroids=force_reassign + save_mat=True, assign_centroids=False ) def plot_imp_map( From 8ed6bbe7d54f9dc33ddd7f825a63986047a8c0e9 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 15:44:03 +0200 Subject: [PATCH 12/16] forecastforecast.calc: there is no implicit centroid assignment in impact.calc anymore --- climada/engine/forecast.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/climada/engine/forecast.py b/climada/engine/forecast.py index 236831405f..a88d34954b 100644 --- a/climada/engine/forecast.py +++ b/climada/engine/forecast.py @@ -292,7 +292,8 @@ def calc(self, force_reassign=False): default is false. """ # calc impact - self.exposure.assign_centroids(self.hazard, overwrite=force_reassign) + if self.hazard: + self.exposure.assign_centroids(self.hazard[0], overwrite=force_reassign) for ind_i, haz_i in enumerate(self.hazard): self._impact[ind_i].calc( self.exposure, self.vulnerability, haz_i, From 4834f793201e745c6b9cb927b97e2926494a216f Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Fri, 5 Aug 2022 18:08:37 +0200 Subject: [PATCH 13/16] minimize centroid assignment in uncertainty, cost_benefit and measures --- climada/engine/cost_benefit.py | 22 +++++++++++++++------- climada/engine/impact_calc.py | 4 ++-- climada/engine/test/test_cost_benefit.py | 4 ++++ climada/engine/unsequa/calc_impact.py | 4 ++-- climada/entity/measures/base.py | 14 +++++++------- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/climada/engine/cost_benefit.py b/climada/engine/cost_benefit.py index d246cc6d96..0a9b597221 100644 --- a/climada/engine/cost_benefit.py +++ b/climada/engine/cost_benefit.py @@ -158,7 +158,7 @@ def __init__(self): self.imp_meas_present = dict() def calc(self, hazard, entity, haz_future=None, ent_future=None, - future_year=None, risk_func=risk_aai_agg, imp_time_depen=None, save_imp=False): + future_year=None, risk_func=risk_aai_agg, imp_time_depen=None, save_imp=False, assign_centroids=True): """Compute cost-benefit ratio for every measure provided current and, optionally, future conditions. Present and future measures need to have the same name. The measures costs need to be discounted by the user. @@ -203,6 +203,14 @@ def calc(self, hazard, entity, haz_future=None, ent_future=None, if future_year is None and ent_future is None: future_year = entity.exposures.ref_year + # assign centroids + if assign_centroids: + entity.exposures.assign_centroids(hazard, overwrite=True) + if ent_future: + ent_future.exposures.assign_centroids( + haz_future if haz_future else hazard, overwrite=True + ) + if not haz_future and not ent_future: self.future_year = future_year self._calc_impact_measures(hazard, entity.exposures, @@ -525,11 +533,11 @@ def plot_waterfall(hazard, entity, haz_future, ent_future, future_year = ent_future.exposures.ref_year imp = Impact() - imp.calc(entity.exposures, entity.impact_funcs, hazard) + imp.calc(entity.exposures, entity.impact_funcs, hazard, assign_centroids=False) curr_risk = risk_func(imp) imp = Impact() - imp.calc(ent_future.exposures, ent_future.impact_funcs, haz_future) + imp.calc(ent_future.exposures, ent_future.impact_funcs, haz_future, assign_centroids=False) fut_risk = risk_func(imp) if not axis: @@ -542,7 +550,7 @@ def plot_waterfall(hazard, entity, haz_future, ent_future, # changing future # socio-economic dev imp = Impact() - imp.calc(ent_future.exposures, ent_future.impact_funcs, hazard) + imp.calc(ent_future.exposures, ent_future.impact_funcs, hazard, assign_centroids=False) risk_dev = risk_func(imp) LOGGER.info('Risk with development at {:d}: {:.3e}'.format(future_year, risk_dev)) @@ -687,7 +695,7 @@ def plot_waterfall_accumulated(self, hazard, entity, ent_future, time_dep = self._time_dependency_array(imp_time_depen) # socio-economic dev imp = Impact() - imp.calc(ent_future.exposures, ent_future.impact_funcs, hazard) + imp.calc(ent_future.exposures, ent_future.impact_funcs, hazard, assign_centroids=False) risk_dev = self._npv_unaverted_impact(risk_func(imp), entity.disc_rates, time_dep, curr_risk) LOGGER.info('Total risk with development at {:d}: {:.3e}'.format( @@ -760,7 +768,7 @@ def _calc_impact_measures(self, hazard, exposures, meas_set, imp_fun_set, # compute impact without measures LOGGER.debug('%s impact with no measure.', when) imp_tmp = Impact() - imp_tmp.calc(exposures, imp_fun_set, hazard) + imp_tmp.calc(exposures, imp_fun_set, hazard, assign_centroids=False) impact_meas[NO_MEASURE] = dict() impact_meas[NO_MEASURE]['cost'] = (0, 0) impact_meas[NO_MEASURE]['risk'] = risk_func(imp_tmp) @@ -772,7 +780,7 @@ def _calc_impact_measures(self, hazard, exposures, meas_set, imp_fun_set, # compute impact for each measure for measure in meas_set.get_measure(hazard.tag.haz_type): LOGGER.debug('%s impact of measure %s.', when, measure.name) - imp_tmp, risk_transf = measure.calc_impact(exposures, imp_fun_set, hazard) + imp_tmp, risk_transf = measure.calc_impact(exposures, imp_fun_set, hazard, assign_centroids=False) impact_meas[measure.name] = dict() impact_meas[measure.name]['cost'] = (measure.cost, measure.risk_transf_cost_factor) impact_meas[measure.name]['risk'] = risk_func(imp_tmp) diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 4aa28636c7..34749aa0b9 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -275,8 +275,8 @@ def minimal_exp_gdf(self, impf_col, assign_centroids): elif self.hazard.centr_exp_col not in self.exposures.gdf.columns: raise ValueError("'assign_centroids' is set to 'False' but no centroids are assigned" f" for the given hazard type ({self.hazard.tag.haz_type})." - " Run `asssign_centroids` beforehand or set the 'assign_centroids'" - " to 'True'") + " Run 'exposures.assign_centroids()' beforehand or set" + " 'assign_centroids' to 'True'") mask = ( (self.exposures.gdf.value.values != 0) & (self.exposures.gdf[self.hazard.centr_exp_col].values >= 0) diff --git a/climada/engine/test/test_cost_benefit.py b/climada/engine/test/test_cost_benefit.py index 8b978e78f8..c4014d9908 100644 --- a/climada/engine/test/test_cost_benefit.py +++ b/climada/engine/test/test_cost_benefit.py @@ -61,6 +61,7 @@ def test_calc_impact_measures_pass(self): for meas in entity.measures.get_measure('TC'): meas.haz_type = 'TC' entity.check() + entity.exposures.assign_centroids(hazard) cost_ben = CostBenefit() cost_ben._calc_impact_measures(hazard, entity.exposures, entity.measures, @@ -241,6 +242,7 @@ def test_calc_cb_no_change_pass(self): for meas in entity.measures.get_measure('TC'): meas.haz_type = 'TC' entity.check() + entity.exposures.assign_centroids(hazard) cost_ben = CostBenefit() cost_ben._calc_impact_measures(hazard, entity.exposures, entity.measures, @@ -277,6 +279,7 @@ def test_calc_cb_change_pass(self): for meas in entity.measures.get_measure('TC'): meas.haz_type = 'TC' entity.check() + entity.exposures.assign_centroids(hazard) cost_ben = CostBenefit() cost_ben._calc_impact_measures(hazard, entity.exposures, entity.measures, @@ -288,6 +291,7 @@ def test_calc_cb_change_pass(self): haz_future = copy.deepcopy(hazard) haz_future.intensity.data += 25 + ent_future.exposures.assign_centroids(haz_future) cost_ben._calc_impact_measures(haz_future, ent_future.exposures, ent_future.measures, ent_future.impact_funcs, when='future', diff --git a/climada/engine/unsequa/calc_impact.py b/climada/engine/unsequa/calc_impact.py index ac45d23439..e24bb8043b 100644 --- a/climada/engine/unsequa/calc_impact.py +++ b/climada/engine/unsequa/calc_impact.py @@ -261,8 +261,8 @@ def _map_impact_calc(self, sample_iterrows): haz = self.haz_input_var.evaluate(**haz_samples) imp = Impact() - - imp.calc(exposures=exp, impact_funcs=impf, hazard=haz) + exp.assign_centroids(haz, overwrite=False) + imp.calc(exposures=exp, impact_funcs=impf, hazard=haz, assign_centroids=False) # Extract from climada.impact the chosen metrics freq_curve = imp.calc_freq_curve(self.rp).impact diff --git a/climada/entity/measures/base.py b/climada/entity/measures/base.py index 9ebd6f865e..f286d8a11b 100755 --- a/climada/entity/measures/base.py +++ b/climada/entity/measures/base.py @@ -121,7 +121,7 @@ def check(self): u_check.size(2, self.mdd_impact, 'Measure.mdd_impact') u_check.size(2, self.paa_impact, 'Measure.paa_impact') - def calc_impact(self, exposures, imp_fun_set, hazard): + def calc_impact(self, exposures, imp_fun_set, hazard, assign_centroids=True): """ Apply measure and compute impact and risk transfer of measure implemented over inputs. @@ -137,12 +137,12 @@ def calc_impact(self, exposures, imp_fun_set, hazard): Returns ------- - : climada.engine.Impact + climada.engine.Impact resulting impact and risk transfer of measure """ new_exp, new_impfs, new_haz = self.apply(exposures, imp_fun_set, hazard) - return self._calc_impact(new_exp, new_impfs, new_haz) + return self._calc_impact(new_exp, new_impfs, new_haz, assign_centroids) def apply(self, exposures, imp_fun_set, hazard): """ @@ -180,7 +180,7 @@ def apply(self, exposures, imp_fun_set, hazard): return new_exp, new_impfs, new_haz - def _calc_impact(self, new_exp, new_impfs, new_haz): + def _calc_impact(self, new_exp, new_impfs, new_haz, assign_centroids): """Compute impact and risk transfer of measure implemented over inputs. Parameters @@ -194,11 +194,11 @@ def _calc_impact(self, new_exp, new_impfs, new_haz): Returns ------- - : climada.engine.Impact + climada.engine.Impact """ from climada.engine.impact import Impact imp = Impact() - imp.calc(new_exp, new_impfs, new_haz) + imp.calc(new_exp, new_impfs, new_haz, assign_centroids=assign_centroids) return imp.calc_risk_transfer(self.risk_transf_attach, self.risk_transf_cover) def _change_all_hazard(self, hazard): @@ -360,7 +360,7 @@ def _cutoff_hazard_damage(self, exposures, impf_set, hazard): from climada.engine.impact import Impact imp = Impact() - imp.calc(exp_imp, impf_set, hazard) + imp.calc(exp_imp, impf_set, hazard, assign_centroids=False) LOGGER.debug('Cutting events whose damage have a frequency > %s.', self.hazard_freq_cutoff) From d764833af8c02236d4e559955b92b9565b74a371 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Mon, 8 Aug 2022 11:19:45 +0200 Subject: [PATCH 14/16] measures.test: explicit centroid assignment before _cutoff --- climada/entity/measures/test/test_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/climada/entity/measures/test/test_base.py b/climada/entity/measures/test/test_base.py index 17e0451cc0..62ff2f31e7 100644 --- a/climada/entity/measures/test/test_base.py +++ b/climada/entity/measures/test/test_base.py @@ -83,6 +83,7 @@ def test_cutoff_hazard_pass(self): exp = Exposures.from_mat(ENT_TEST_MAT) exp.gdf.rename(columns={'impf': 'impf_TC'}, inplace=True) exp.check() + exp.assign_centroids(haz) imp_set = ImpactFuncSet.from_mat(ENT_TEST_MAT) @@ -117,6 +118,7 @@ def test_cutoff_hazard_region_pass(self): exp.gdf['region_id'] = np.zeros(exp.gdf.shape[0]) exp.gdf.region_id.values[10:] = 1 exp.check() + exp.assign_centroids(haz) imp_set = ImpactFuncSet.from_mat(ENT_TEST_MAT) From b7962ffe62569a3c8d1e5a1fc2bbb28ff49f0427 Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Tue, 9 Aug 2022 16:30:47 +0200 Subject: [PATCH 15/16] unsequa.calc_cost_benefit: assign centroids manually, w/o overwriting analagous to the assignment policy in unsequa.calc_impact --- climada/engine/cost_benefit.py | 7 ++++--- climada/engine/unsequa/calc_cost_benefit.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/climada/engine/cost_benefit.py b/climada/engine/cost_benefit.py index 0a9b597221..72ca207ca5 100644 --- a/climada/engine/cost_benefit.py +++ b/climada/engine/cost_benefit.py @@ -157,8 +157,8 @@ def __init__(self): self.imp_meas_future = dict() self.imp_meas_present = dict() - def calc(self, hazard, entity, haz_future=None, ent_future=None, - future_year=None, risk_func=risk_aai_agg, imp_time_depen=None, save_imp=False, assign_centroids=True): + def calc(self, hazard, entity, haz_future=None, ent_future=None, future_year=None, + risk_func=risk_aai_agg, imp_time_depen=None, save_imp=False, assign_centroids=True): """Compute cost-benefit ratio for every measure provided current and, optionally, future conditions. Present and future measures need to have the same name. The measures costs need to be discounted by the user. @@ -780,7 +780,8 @@ def _calc_impact_measures(self, hazard, exposures, meas_set, imp_fun_set, # compute impact for each measure for measure in meas_set.get_measure(hazard.tag.haz_type): LOGGER.debug('%s impact of measure %s.', when, measure.name) - imp_tmp, risk_transf = measure.calc_impact(exposures, imp_fun_set, hazard, assign_centroids=False) + imp_tmp, risk_transf = measure.calc_impact(exposures, imp_fun_set, hazard, + assign_centroids=False) impact_meas[measure.name] = dict() impact_meas[measure.name]['cost'] = (measure.cost, measure.risk_transf_cost_factor) impact_meas[measure.name]['risk'] = risk_func(imp_tmp) diff --git a/climada/engine/unsequa/calc_cost_benefit.py b/climada/engine/unsequa/calc_cost_benefit.py index 18aff4ef3c..0ca08db844 100644 --- a/climada/engine/unsequa/calc_cost_benefit.py +++ b/climada/engine/unsequa/calc_cost_benefit.py @@ -279,8 +279,11 @@ def _map_costben_calc(self, param_sample, **kwargs): ent_fut = self.ent_fut_input_var.evaluate(**ent_fut_samples) cb = CostBenefit() + ent.exposures.assign_centroids(haz, overwrite=False) + if ent_fut: + ent_fut.exposures.assign_centroids(haz_fut if haz_fut else haz, overwrite=False) cb.calc(hazard=haz, entity=ent, haz_future=haz_fut, ent_future=ent_fut, - save_imp=False, **kwargs) + save_imp=False, assign_centroids=False, **kwargs) # Extract from climada.impact the chosen metrics return [cb.imp_meas_present, From 8c5ebe075189c87ea32b4720302f1edca9a99b0d Mon Sep 17 00:00:00 2001 From: emanuel-schmid Date: Wed, 10 Aug 2022 00:09:24 +0200 Subject: [PATCH 16/16] docstring consolidation --- climada/engine/cost_benefit.py | 9 ++++++++- climada/engine/impact_calc.py | 14 ++++++-------- climada/entity/measures/base.py | 6 ++++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/climada/engine/cost_benefit.py b/climada/engine/cost_benefit.py index 72ca207ca5..794525e9ec 100644 --- a/climada/engine/cost_benefit.py +++ b/climada/engine/cost_benefit.py @@ -188,7 +188,14 @@ def calc(self, hazard, entity, haz_future=None, ent_future=None, future_year=Non count the same when there is no future hazard nor entity and 1 (linear annual change) when there is future hazard or entity. Default is None. - save_imp : bool, optional) + save_imp : bool, optional + Default: False + assign_centroids : bool, optional + indicates whether centroids are assigned to the self.exposures object. + Centroids assignment is an expensive operation; set this to ``False`` to save + computation time if the exposures from ``ent`` and ``ent_fut`` have already + centroids assigned for the respective hazards. + Default: True True if Impact of each measure is saved. Default is False. """ # Present year given in entity. Future year in ent_future if provided. diff --git a/climada/engine/impact_calc.py b/climada/engine/impact_calc.py index 34749aa0b9..95f0e82597 100644 --- a/climada/engine/impact_calc.py +++ b/climada/engine/impact_calc.py @@ -111,11 +111,10 @@ def impact(self, save_mat=True, assign_centroids=True): if true, save the total impact matrix (events x exposures) Default: True assign_centroids : bool, optional - indicates whether centroids are re-assigned to the self.exposures object - or kept from previous impact calculation with a hazard of the same hazard type. + indicates whether centroids are assigned to the self.exposures object. Centroids assignment is an expensive operation; set this to ``False`` to save - computation time if the centroids have not changed since the last impact - calculation. + computation time if the hazards' centroids are already assigned to the exposures + object. Default: True Examples @@ -157,11 +156,10 @@ def insured_impact(self, save_mat=False, assign_centroids=True): save_mat : bool if true, save the total impact matrix (events x exposures) assign_centroids : bool, optional - indicates whether centroids are re-assigned to the self.exposures object - or kept from previous impact calculation with a hazard of the same hazard type. + indicates whether centroids are assigned to the self.exposures object. Centroids assignment is an expensive operation; set this to ``False`` to save - computation time if the centroids have not changed since the last impact - calculation. + computation time if the hazards' centroids are already assigned to the exposures + object. Default: True Examples diff --git a/climada/entity/measures/base.py b/climada/entity/measures/base.py index f286d8a11b..4113e20c47 100755 --- a/climada/entity/measures/base.py +++ b/climada/entity/measures/base.py @@ -134,6 +134,12 @@ def calc_impact(self, exposures, imp_fun_set, hazard, assign_centroids=True): impact function set instance hazard : climada.hazard.Hazard hazard instance + assign_centroids : bool, optional + indicates whether centroids are assigned to the self.exposures object. + Centroids assignment is an expensive operation; set this to ``False`` to save + computation time if the hazards' centroids are already assigned to the exposures + object. + Default: True Returns -------