From 1c5e15b6c5ed7d08dc215d5ae925a5550fb32818 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Thu, 1 Aug 2024 13:46:05 +0200 Subject: [PATCH 01/43] add file for depth averaging with sigma layers --- .../test_depth_average_sigma_rule.nc | Bin 0 -> 17715 bytes ...age_rule.nc => test_depth_average_z_rule.nc} | Bin .../test17_depth_average_rule.yaml | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc rename tests_acceptance/input_nc_files/{test_depth_average_rule.nc => test_depth_average_z_rule.nc} (100%) diff --git a/tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc b/tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc new file mode 100644 index 0000000000000000000000000000000000000000..3b9e110230828dee8ae56964425ebc58d7bf0dc0 GIT binary patch literal 17715 zcmeHP4Qv$06@GiWb`G;P#x}cA&pzrq z2&R=tMO(F@jU`$o5k!epB%(&4O4Bp~5e1s`C#q5jg(^`hwW<0eMO8`yjT$Eu-1lbY zt=DTW+$OexW=Fo8nKy4{-hA`+&CJ_5f3K-AT3qy*qChYR97a{Bj!4Q%RnT@|V6?rd zu_ItU8(1W0`T|w7y+kJdtKvm-WWFc>8l~pl1O`UB2HdzaHbrMTlt7SJV{PG0gP9%4 z!CWetg->B!>M?+yUCF_0Dl<^Ks$w9S>q+$1)>PhIxvHWs(R){IRb^F8rx+QwCq~xT+!|@?Y;JFhb%yI(B4Bm_%A#Xu0+P1+)`;M* zQ~aCzQrQsx#Y2chxG@?^r23Ns1Wu@XXD&35$&D55iVvpZ-RWdyNQJ7EV4^?)Y`+9d z3Aw!&@6i=73ud2R?vbo^cFxt5ri#$)PiFg8C1L}aL^3ucn%DCMmgKomX+ANP zz#h69z!pv;u*t->WNf!LU`d|%n=d{#mB2o6HGsWgx`Fkmu_RB!pC5c{DuMm^)c}@G zEz@NRdgDEwHJ0T0v0?sH0&Cx#c(<`~)jLRs9}IK^p)ez>-Yo3MGpv)XdS!L{mjdhw znkS+iO`Cx9E&yULL}M9S@D8b+5$|E}$RmSePdE9|PVw3ztY)2BT1~B5`}_-i)WVr?Mp+@b>`{i@Ty2=AADdP z%){1ig2zVYG{8KhE`Vo;&96fXCU1dbBg5Z;3R&&!rNhvV{oM%f&0703lwmRiTTdT( z0jjWtPs6f(TV8_mg7?(M#lM1nOx_Ade*4I2SS5IO-TRe4z+p_@2G3U%z5yMWyb=C% z{F~?C8By(>Cy$T7cLnv%AKZKiUcuz8kZxE*a{1w-At2_T7C;%~6eF-Q$ho!jOU^o* z7&V3L(ohpdN-ly_29vXL;)*#dC)(P;%DGfe23jAXbk+kbHOP7>rJ|N`Q&KyW{Q{-C z!t+8L=kXH~sw?=VUd|V3{F18o=JUOuWy&Wkwr~4SI+oTv_p)LE^SCo0%l}B}fo_ag zovwV0=FCxYg|7~3W<$*=vK5rjVxh0#1rcCmuiP0NR;$@PvT18)a2eN6GqHL) z4?6{=2FMDL=`nH1qjT|WJY3W9I+{wSzZg&NOv2nr1~S>{np&d)nTx1J9cD>aXFAgp z&!HbvxI1YFK)ulc!VHo4mP?e*Y=VdEB=N{YoSCMjR>uXql;j%otF|7_VCA`7mR(a~)p)w5A0tKd5ljs=K z9>}8Kry3wNqrm*!4l5JWxgECjOy_pkiZh+tVRO!OZinqU)43fs@Jv6U2E>-0>D&%) z1DMYJvax46^UKkq|H@j0nBvDi0zLvh0zLvh0zLvh0zLvh0zLvh0zLvh0#_dayUWgU za4E6O@5?v!b6_R;BswC|NJpeC9Ek~^#A@^a&?!6-&&9LJ+?aF1y3z$fDwhZC(4;<# zs5)jVX9DQLwI=-{B*as9(Dzt=&p-O2mzDS_!1p!eMVTsz*}uUtODudQC>cx4d6t;- zXKTd(tgkKWWs=L+MA!d&@|!T)a~fjt}xgQ_TODS*MRC*~{E zQVd!wo7&bN|F*gsq#5ioVt9MgnfsFIWPhwX?NlFyfpp$h1hg6=OuzG%o#kh$r2u@F zhn?lV`*up*x1{s*(6T$1RMVa(ee`^d>cqKUR@iN4_HJwi1A9`bLGnQ7b+wi(!t1Xb z`;TZreepxVv=_?1_0<_w)V0H>SWyqJmQq~rii$(Jmftgwciyg;q)YZZtqGnLb#(wrxcYx^MVi$n-Mb?vo4?DflnYvvDNlAfYUK=D8 zM{uNYN_?@vEzZ>0IpYv|~Zgj5ajNSO%=+63_cf-?3A5r(pob)T{ zzmwl@2Sjkg&Btd(@WC}+_T$t~w|Q})`|Z?ScD{UkJ{CT-3%ZD(57g&Fm6w?F_g_xR zRPYp2Wc>d_=RF$50X4{vlOB)LCo^Z?Z4ayM{<{haPuqQ}v*lv*Ff}M|oc*;?wsQ|R zx16TUx{hW2LY15_G@b0?65cD!R}SA^t-O#t^ZShQ5_zz_*Y<-XLv_G$zCRILr&?zL z*{4$AegUzby7F`-jz#B~&hK=vq^ofGTwt>yMgUEAfvtrPMfqM}V_EDmdpxAGaL=Qe zuT2D+y^&U&=Wp@;ey5zF83mdKcP#@gy{?P5FWVu3{+XsNA(#Sq#jr@?g(@P0GyhR8 zfSy?=+ww385vt)y+u1C0Obv*MKwB`wmp{Mc^{HT)IVQRJsUR1(JwKlY1}7TcueKXd zT$e>aLnngyDBxY?sdQJciQ||(<*GyLr3mNQC(e0(3BvhO{pP0euQYJ<{~YWxHG@1% z6lN*AoUP)@n!)~ Date: Thu, 1 Aug 2024 14:04:26 +0200 Subject: [PATCH 02/43] working version for sigma layers. DOES not work for z --- .../entities/rules/depth_average_rule.py | 43 ++++++++++++++++++- .../data/parsers/parser_depth_average_rule.py | 3 +- .../test17_depth_average_rule.yaml | 2 +- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 2c6c8124..690222f3 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -13,6 +13,7 @@ from typing import Dict import xarray as _xr +from regex import I from decoimpact.business.entities.rules.i_multi_array_based_rule import ( IMultiArrayBasedRule, @@ -20,6 +21,7 @@ from decoimpact.business.entities.rules.rule_base import RuleBase from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, + INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, ) @@ -46,8 +48,10 @@ def execute( # just used the first value. variables = next(iter(value_arrays.values())) + layer_type_suffix = self.determine_layer_type(logger) + depths_interfaces = self._extract_variable_based_on_suffix( - value_arrays, INTERFACES_Z_SUFFIX) + value_arrays, layer_type_suffix) water_level_values = self._extract_variable_based_on_suffix( value_arrays, WATER_LEVEL_SUFFIX) bed_level_values = self._extract_variable_based_on_suffix( @@ -126,3 +130,40 @@ def _extract_variable_based_on_suffix( """ variable = [value_arrays[name] for name in value_arrays if suffix in name][0] return variable + + def determine_layer_type( + self, + logger: ILogger + ): + """Determine whether the model is a sigma or Z layer model based on which variable is present for the interfaces. Give error if both are found. + + Args: + value_array (DataArray): Values + + Returns: + layer_type (str): sigma or z + """ + has_z_layers = self._check_if_layer_type_is_present("z") + has_sigma_layers = self._check_if_layer_type_is_present("sigma") + if has_sigma_layers and has_z_layers: + logger.log_error("Both Z and Sigma layers are present. This combination is not supported for depth averaging.") + if has_sigma_layers: + return INTERFACES_SIGMA_SUFFIX + if has_z_layers: + return INTERFACES_Z_SUFFIX + + + + def _check_if_layer_type_is_present( + self, + layer_type: str, + ): + """Determine whether a layer type is present. + + Args: + layer_type (str): sigma or z + + Returns: + bool + """ + return any("interface_" + layer_type in variable for variable in self._input_variable_names) diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 93df79a5..e9054129 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -14,6 +14,7 @@ from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, + INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, ) @@ -43,7 +44,7 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: name: str = get_dict_element("name", dictionary) input_variable_names: List[str] = [ get_dict_element("input_variable", dictionary), - INTERFACES_Z_SUFFIX, + INTERFACES_SIGMA_SUFFIX, WATER_LEVEL_SUFFIX, BED_LEVEL_SUFFIX, ] diff --git a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml b/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml index d224caeb..6e43d88e 100644 --- a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml +++ b/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml @@ -2,7 +2,7 @@ version: 0.0.0 input-data: - dataset: - filename: ./tests_acceptance/input_nc_files/test_depth_average_z_rule.nc + filename: ./tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc rules: - depth_average_rule: From dd9896271b3f53b1a00dc42ab06282b9452d5282 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 09:58:38 +0200 Subject: [PATCH 03/43] add test 18, correct paths for test 17 --- ... => test_depth_average_sigmalayer_rule.nc} | Bin ...e.nc => test_depth_average_zlayer_rule.nc} | Bin ... => test17_depth_average_zlayer_rule.yaml} | 4 +-- .../test18_depth_average_sigmalayer_rule.yaml | 27 ++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) rename tests_acceptance/input_nc_files/{test_depth_average_sigma_rule.nc => test_depth_average_sigmalayer_rule.nc} (100%) rename tests_acceptance/input_nc_files/{test_depth_average_z_rule.nc => test_depth_average_zlayer_rule.nc} (100%) rename tests_acceptance/input_yaml_files/{test17_depth_average_rule.yaml => test17_depth_average_zlayer_rule.yaml} (93%) create mode 100644 tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml diff --git a/tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc b/tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc similarity index 100% rename from tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc rename to tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc diff --git a/tests_acceptance/input_nc_files/test_depth_average_z_rule.nc b/tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc similarity index 100% rename from tests_acceptance/input_nc_files/test_depth_average_z_rule.nc rename to tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc diff --git a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml similarity index 93% rename from tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml rename to tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml index 6e43d88e..aa5edb9f 100644 --- a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml +++ b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml @@ -2,7 +2,7 @@ version: 0.0.0 input-data: - dataset: - filename: ./tests_acceptance/input_nc_files/test_depth_average_sigma_rule.nc + filename: ./tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc rules: - depth_average_rule: @@ -24,4 +24,4 @@ rules: output_variable: output_C output-data: - filename: ./tests_acceptance/output_nc_files/test17_depth_average_rule.nc + filename: ./tests_acceptance/output_nc_files/test17_depth_average_zlayer_rule.nc diff --git a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml new file mode 100644 index 00000000..edfff22d --- /dev/null +++ b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml @@ -0,0 +1,27 @@ +version: 0.0.0 + +input-data: + - dataset: + filename: ./tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc + +rules: + - depth_average_rule: + name: Test A + description: Test A + input_variable: var_3d_A + output_variable: output_A + + - depth_average_rule: + name: Test B + description: Test B + input_variable: var_3d_B + output_variable: output_B + + - depth_average_rule: + name: Test C + description: Test C + input_variable: var_3d_C + output_variable: output_C + +output-data: + filename: ./tests_acceptance/output_nc_files/test_depth_average_sigmalayer_rule.nc From 782479c3ffcf2c22c6540c26c417bc71f115ec95 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 10:00:05 +0200 Subject: [PATCH 04/43] rename test 17 reference file --- ..._rule.nc => test17_depth_average_zlayer_rule.nc} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename tests_acceptance/reference_nc_files/{test17_depth_average_rule.nc => test17_depth_average_zlayer_rule.nc} (100%) diff --git a/tests_acceptance/reference_nc_files/test17_depth_average_rule.nc b/tests_acceptance/reference_nc_files/test17_depth_average_zlayer_rule.nc similarity index 100% rename from tests_acceptance/reference_nc_files/test17_depth_average_rule.nc rename to tests_acceptance/reference_nc_files/test17_depth_average_zlayer_rule.nc From 1e5807651b8e0f1fa8406167c781f46b689d9c33 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 10:03:17 +0200 Subject: [PATCH 05/43] flake problems --- decoimpact/business/entities/rules/depth_average_rule.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 690222f3..d11e09d5 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -152,8 +152,6 @@ def determine_layer_type( if has_z_layers: return INTERFACES_Z_SUFFIX - - def _check_if_layer_type_is_present( self, layer_type: str, @@ -166,4 +164,5 @@ def _check_if_layer_type_is_present( Returns: bool """ - return any("interface_" + layer_type in variable for variable in self._input_variable_names) + return any("interface_" + layer_type + in variable for variable in self._input_variable_names) From 140108741267c8924008edd0c82e0ed0d76ee58c Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 10:04:04 +0200 Subject: [PATCH 06/43] more flake problems --- decoimpact/business/entities/rules/depth_average_rule.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index d11e09d5..99f7297c 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -146,7 +146,9 @@ def determine_layer_type( has_z_layers = self._check_if_layer_type_is_present("z") has_sigma_layers = self._check_if_layer_type_is_present("sigma") if has_sigma_layers and has_z_layers: - logger.log_error("Both Z and Sigma layers are present. This combination is not supported for depth averaging.") + logger.log_error("Both Z and Sigma layers are present. This" + "combination is not supported for depth" + "averaging.") if has_sigma_layers: return INTERFACES_SIGMA_SUFFIX if has_z_layers: From 2dcd52a44640f30332d1043cf3bb9e63fb0dca24 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 10:04:24 +0200 Subject: [PATCH 07/43] and more flake8 --- decoimpact/business/entities/rules/depth_average_rule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 99f7297c..bd3ba6d5 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -135,7 +135,8 @@ def determine_layer_type( self, logger: ILogger ): - """Determine whether the model is a sigma or Z layer model based on which variable is present for the interfaces. Give error if both are found. + """Determine whether the model is a sigma or Z layer model based on which + variable is present for the interfaces. Give error if both are found. Args: value_array (DataArray): Values From 66886c2298e20c431c05871e0fd5a3a541dc4c1e Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Fri, 2 Aug 2024 13:14:58 +0200 Subject: [PATCH 08/43] adding print statements in an attempt to get both to work --- decoimpact/business/entities/rule_based_model.py | 4 ++++ decoimpact/business/entities/rule_processor.py | 5 +++++ decoimpact/business/utils/dataset_utils.py | 8 ++++++++ decoimpact/crosscutting/delft3d_specific_data.py | 2 ++ decoimpact/data/parsers/parser_depth_average_rule.py | 5 ++++- 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 9f1eb138..7b47f1cc 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -149,6 +149,7 @@ def _make_output_variables_list(self) -> list: all_input_variables = _lu.flatten_list(list(all_inputs.values())) all_vars = var_list + mapping_keys + all_input_variables + print('sssss', all_vars) return _lu.remove_duplicates_from_list(all_vars) @@ -219,6 +220,7 @@ def _get_direct_rule_inputs(self, rule_names) -> Dict[str, List[str]]: Dict[str, List[str]] """ rule_input_vars = [rule.input_variable_names for rule in self._rules] + print('check',rule_input_vars) rule_output_vars = [rule.output_variable_name for rule in self._rules] needed_input_per_rule = {} @@ -226,6 +228,7 @@ def _get_direct_rule_inputs(self, rule_names) -> Dict[str, List[str]]: needed_input_per_rule[rule_names[index]] = _lu.items_not_in( inputs_per_rule, rule_output_vars ) + print('check2',needed_input_per_rule) return needed_input_per_rule @@ -234,6 +237,7 @@ def _extend_names(self, dummy_variable_name: str): when using hardcoded Delftd3D names. """ for rule in self._rules: + print('rule',rule.input_variable_names) rule.input_variable_names = _du.extend_to_full_name( rule.input_variable_names, dummy_variable_name diff --git a/decoimpact/business/entities/rule_processor.py b/decoimpact/business/entities/rule_processor.py index 6c70735c..be65d424 100644 --- a/decoimpact/business/entities/rule_processor.py +++ b/decoimpact/business/entities/rule_processor.py @@ -64,10 +64,13 @@ def initialize(self, logger: ILogger) -> bool: bool: A boolean to indicate if all the rules can be processed. """ inputs: List[str] = [] + print('ini',self._input_dataset) inputs = _lu.flatten_list( [_du.list_vars(self._input_dataset), _du.list_coords(self._input_dataset)] ) + print('ini',inputs) + print('ini',self._input_dataset) tree, success = self._create_rule_sets(inputs, self._rules, [], logger) if success: self._processing_list = tree @@ -169,6 +172,8 @@ def _get_solvable_rules( for rule in unprocessed_rules: names = rule.input_variable_names + print('QQQ names',names) + print('QQQ inputs',inputs) if all(name in inputs for name in names): solvable_rules.append(rule) diff --git a/decoimpact/business/utils/dataset_utils.py b/decoimpact/business/utils/dataset_utils.py index 414477fa..e45e0d5f 100644 --- a/decoimpact/business/utils/dataset_utils.py +++ b/decoimpact/business/utils/dataset_utils.py @@ -6,9 +6,11 @@ # https://github.com/Deltares/D-EcoImpact/blob/main/LICENSE.md """Library for utility functions regarding an xarray dataset""" +from re import I from typing import List, Optional import xarray as _xr +from numpy import var import decoimpact.business.utils.list_utils as _lu from decoimpact.crosscutting.delft3d_specific_data import delft3d_specific_names @@ -78,6 +80,10 @@ def remove_all_variables_except( dummy_dependent_var_list = get_dummy_and_dependent_var_list(dataset) variables_to_keep += dummy_dependent_var_list + # correct interface name based on Z or Sigma layers + replacement = "_interface_z" + variables_to_keep = list(map(lambda x: x.replace('_interface', f'_{replacement}') if isinstance(x, str) and '_interface' in x else x, variables_to_keep)) + print('llll',variables_to_keep) all_variables = list_vars(dataset) variables_to_remove = [ @@ -374,6 +380,8 @@ def extend_to_full_name( Returns: list[str]: list of the extended variable names """ + print('test',variables) variables = [dummy_variable + var if var in delft3d_specific_names else var for var in variables] + print('test',variables) return variables diff --git a/decoimpact/crosscutting/delft3d_specific_data.py b/decoimpact/crosscutting/delft3d_specific_data.py index c0ecfb1b..2f9ff3be 100644 --- a/decoimpact/crosscutting/delft3d_specific_data.py +++ b/decoimpact/crosscutting/delft3d_specific_data.py @@ -8,6 +8,7 @@ Configuration file for hardcoded delft3d variable names """ +INTERFACES_GENERIC_SUFFIX = "_interface" INTERFACES_Z_SUFFIX = "_interface_z" INTERFACES_SIGMA_SUFFIX = "_interface_sigma" BED_LEVEL_SUFFIX = "_flowelem_bl" @@ -15,6 +16,7 @@ delft3d_specific_names = [ + INTERFACES_GENERIC_SUFFIX, INTERFACES_Z_SUFFIX, INTERFACES_SIGMA_SUFFIX, BED_LEVEL_SUFFIX, diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index e9054129..75e072b1 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -14,6 +14,7 @@ from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, + INTERFACES_GENERIC_SUFFIX, INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, @@ -44,7 +45,9 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: name: str = get_dict_element("name", dictionary) input_variable_names: List[str] = [ get_dict_element("input_variable", dictionary), - INTERFACES_SIGMA_SUFFIX, + INTERFACES_GENERIC_SUFFIX, + #INTERFACES_SIGMA_SUFFIX, + #INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, BED_LEVEL_SUFFIX, ] From 831f072b6b07f71ee20f740ea27757a375528eea Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 00:42:12 +0200 Subject: [PATCH 09/43] Go for the much simpler approach. Let the user specify the layer type --- .../business/entities/rule_based_model.py | 4 --- .../business/entities/rule_processor.py | 6 +--- decoimpact/business/utils/dataset_utils.py | 6 ---- .../crosscutting/delft3d_specific_data.py | 2 -- .../data/parsers/parser_depth_average_rule.py | 29 +++++++++++++++--- .../test_data_access_layer_data/results.nc | Bin 6156 -> 6156 bytes 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index e2a41a85..294361dd 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -150,7 +150,6 @@ def _make_output_variables_list(self) -> list: all_input_variables = _lu.flatten_list(list(all_inputs.values())) all_vars = var_list + mapping_keys + all_input_variables - print('sssss', all_vars) return _lu.remove_duplicates_from_list(all_vars) @@ -221,7 +220,6 @@ def _get_direct_rule_inputs(self, rule_names) -> Dict[str, List[str]]: Dict[str, List[str]] """ rule_input_vars = [rule.input_variable_names for rule in self._rules] - print('check',rule_input_vars) rule_output_vars = [rule.output_variable_name for rule in self._rules] needed_input_per_rule = {} @@ -229,7 +227,6 @@ def _get_direct_rule_inputs(self, rule_names) -> Dict[str, List[str]]: needed_input_per_rule[rule_names[index]] = _lu.items_not_in( inputs_per_rule, rule_output_vars ) - print('check2',needed_input_per_rule) return needed_input_per_rule @@ -238,7 +235,6 @@ def _extend_names(self, dummy_variable_name: str): when using hardcoded Delftd3D names. """ for rule in self._rules: - print('rule',rule.input_variable_names) rule.input_variable_names = _du.extend_to_full_name( rule.input_variable_names, dummy_variable_name diff --git a/decoimpact/business/entities/rule_processor.py b/decoimpact/business/entities/rule_processor.py index be65d424..dcc950c6 100644 --- a/decoimpact/business/entities/rule_processor.py +++ b/decoimpact/business/entities/rule_processor.py @@ -64,13 +64,11 @@ def initialize(self, logger: ILogger) -> bool: bool: A boolean to indicate if all the rules can be processed. """ inputs: List[str] = [] - print('ini',self._input_dataset) inputs = _lu.flatten_list( [_du.list_vars(self._input_dataset), _du.list_coords(self._input_dataset)] ) - print('ini',inputs) - print('ini',self._input_dataset) + tree, success = self._create_rule_sets(inputs, self._rules, [], logger) if success: self._processing_list = tree @@ -172,8 +170,6 @@ def _get_solvable_rules( for rule in unprocessed_rules: names = rule.input_variable_names - print('QQQ names',names) - print('QQQ inputs',inputs) if all(name in inputs for name in names): solvable_rules.append(rule) diff --git a/decoimpact/business/utils/dataset_utils.py b/decoimpact/business/utils/dataset_utils.py index b8b83cf8..b8af1b9a 100644 --- a/decoimpact/business/utils/dataset_utils.py +++ b/decoimpact/business/utils/dataset_utils.py @@ -81,10 +81,6 @@ def remove_all_variables_except( dependent_var_list = get_dependent_var_list(dataset, dummy_var) variables_to_keep += dummy_var + dependent_var_list - # correct interface name based on Z or Sigma layers - replacement = "_interface_z" - variables_to_keep = list(map(lambda x: x.replace('_interface', f'_{replacement}') if isinstance(x, str) and '_interface' in x else x, variables_to_keep)) - print('llll',variables_to_keep) all_variables = list_vars(dataset) variables_to_remove = [ @@ -378,9 +374,7 @@ def extend_to_full_name( Returns: list[str]: list of the extended variable names """ - print('test',variables) dummy_variable = dummy_variable[0] variables = [dummy_variable + var if var in delft3d_specific_names else var for var in variables] - print('test',variables) return variables diff --git a/decoimpact/crosscutting/delft3d_specific_data.py b/decoimpact/crosscutting/delft3d_specific_data.py index 0fb243df..73e4dc69 100644 --- a/decoimpact/crosscutting/delft3d_specific_data.py +++ b/decoimpact/crosscutting/delft3d_specific_data.py @@ -9,7 +9,6 @@ """ BED_LEVEL_SUFFIX = "_flowelem_bl" -INTERFACES_GENERIC_SUFFIX = "_interface" INTERFACES_SIGMA_SUFFIX = "_interface_sigma" INTERFACES_Z_SUFFIX = "_interface_z" WATER_LEVEL_SUFFIX = "_s1" @@ -17,7 +16,6 @@ delft3d_specific_names = [ BED_LEVEL_SUFFIX, - INTERFACES_GENERIC_SUFFIX, INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 75e072b1..24da0234 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -14,7 +14,6 @@ from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, - INTERFACES_GENERIC_SUFFIX, INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, @@ -43,11 +42,12 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: RuleBase: Rule based on the provided data """ name: str = get_dict_element("name", dictionary) + layer_type: str = get_dict_element("layer_type", dictionary) + interface_suffix = self._obtain_interface_suffix(layer_type, logger) + input_variable_names: List[str] = [ get_dict_element("input_variable", dictionary), - INTERFACES_GENERIC_SUFFIX, - #INTERFACES_SIGMA_SUFFIX, - #INTERFACES_Z_SUFFIX, + interface_suffix, WATER_LEVEL_SUFFIX, BED_LEVEL_SUFFIX, ] @@ -61,3 +61,24 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: rule_data.description = description return rule_data + + def _obtain_interface_suffix(self, layer_type: str, logger: ILogger): + """Obtain the interface variable based on the layer_type specified. + Give an error if layer_type is not recognised + + Args: + layer_type (str): z or sigma layers + + Returns: + interface_suffix: Suffix for z or sigma layers based on Delft3D + defined suffixes + """ + if layer_type.lower() == 'z': + interface_suffix = INTERFACES_Z_SUFFIX + elif layer_type.lower() == 'sigma': + interface_suffix = INTERFACES_SIGMA_SUFFIX + else: + logger.log_error(f"Layer_type '{layer_type}' is not recognized. Supported " + f"options are '{INTERFACES_Z_SUFFIX}' and " + f"'{INTERFACES_SIGMA_SUFFIX}'.") + return interface_suffix diff --git a/tests/data/entities/test_data_access_layer_data/results.nc b/tests/data/entities/test_data_access_layer_data/results.nc index c443c877abe03fdae015806969385647e3f2f714..07180be2125d9c6543a0a887e1be419d51da9fc7 100644 GIT binary patch delta 59 zcmeA%=rP!!#;RwelaZ2UYHO%xXrgD#Ce8o^j4Vtb3J6$AGILYkL)Z+LC-QP_4q@FP F1OR3b3%~#X delta 59 zcmeA%=rP!!#;RwilaZ2UYHO%xXryPvCe8o^j4Vtb3J6$AGILYkL)Z)-C$-+%9KyOo F2mo%P4I=;m From 6f3005e0ded9e898c42ca281fe729424a2de0c33 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 00:42:28 +0200 Subject: [PATCH 10/43] Update tests to specify layer type --- .../input_yaml_files/test17_depth_average_zlayer_rule.yaml | 3 +++ .../input_yaml_files/test18_depth_average_sigmalayer_rule.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml index aa5edb9f..47c5c699 100644 --- a/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml +++ b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml @@ -9,18 +9,21 @@ rules: name: Test A description: Test A input_variable: var_3d_A + layer_type: Z output_variable: output_A - depth_average_rule: name: Test B description: Test B input_variable: var_3d_B + layer_type: z output_variable: output_B - depth_average_rule: name: Test C description: Test C input_variable: var_3d_C + layer_type: z output_variable: output_C output-data: diff --git a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml index edfff22d..15a7efa4 100644 --- a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml +++ b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml @@ -9,18 +9,21 @@ rules: name: Test A description: Test A input_variable: var_3d_A + layer_type: SIGMA output_variable: output_A - depth_average_rule: name: Test B description: Test B input_variable: var_3d_B + layer_type: sigma output_variable: output_B - depth_average_rule: name: Test C description: Test C input_variable: var_3d_C + layer_type: sigma output_variable: output_C output-data: From 199c43d172202cedaad822c6abaab6a6d1550200 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 00:43:30 +0200 Subject: [PATCH 11/43] add reference file for test 18 --- .../test18_depth_average_sigmalayer_rule.yaml | 2 +- .../test18_depth_average_sigmalayer_rule.nc | Bin 0 -> 22598 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc diff --git a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml index 15a7efa4..3d499f16 100644 --- a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml +++ b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml @@ -27,4 +27,4 @@ rules: output_variable: output_C output-data: - filename: ./tests_acceptance/output_nc_files/test_depth_average_sigmalayer_rule.nc + filename: ./tests_acceptance/output_nc_files/test18_depth_average_sigmalayer_rule.nc diff --git a/tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc b/tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc new file mode 100644 index 0000000000000000000000000000000000000000..b5ca6f6a8cc5a5a3c4dbf72491c0f876b2cea39b GIT binary patch literal 22598 zcmeGk3vg7`^}O9p2x0jeNg+^H0Syph5`v$h)O?o&LimJ&rP5`|ZeVq@OOuT?+793+ ztv{%CoZ6}#svTRM*3k~7Vy&fvwJ^mFermB*2gXjvaYodRPGhaobMCq4@$!;KcGFFe z`Yt5r-E+^q@7#0FJ@?*o@5?O>b-}UuQ}aD}c_8VePQfb}Q>F{Ldg_m^XsBE5@$d1R zuW;U}y6A)P{P3*_41ca+z6T7#Oi=lc)HA>mCIbIVdhkLX{__E>jdpa#k_k8mk-7Ne z!Jo3y1*PSc{`r9IcwsCPE{P_h9pO|o64=zOsK;pPy7~33$;S4r;no!R5!WYOcHru2 z#r{|@i~=h17>1FrX#{!;h~tUp%^;321)wkw3b)}?j_QjypZjvOSQW!v4B*^ps)9Ek zM0|GWpGTx#?3(IrZBunii{hSSLw&7lTwdSY(zv2I)KXjBRIi0rLq(XjFaT=CgUyV? z9wLjO=4h%l(pFg)YmY@>0%}qC*7X1>`=uLK9_d<9+ptWy_BwLSz1=yNn`n`iSd*_G zc&LV&@vmq7d_^v9V@LtTTZB!+}IJRa+MfL}KmH1SvbPsVfyoBvYq~ zt_gRHn5OEkn!^?CsvF}dgdwWVU{wrFbxOhmsP6i; zsY$_>mU?(mk1+$s-g`uC>@{F1iTl|dZI5c%3<`!_eSrlq>8n#8$)5WD=#Q%^D9PzOGPA_|i zsK0LM!8FF8pJqSnERvH?81G!tHa=|UelOIbZAL65nMY2e!)l`QLIF-&IQh!7Rhvw9 zL}Ce?%sczMmZoNWok{|U1@gqS5=eUz9ccC|nBVeDu%WRAWZ6QmIj*O;>+01@`x)cc z{up0NiprZrOwfF*G7N zAs3mkPBIxK-KzPt1xx&#cw)>~vMJc^Ga*=701ua7qmj+zeloIH;^&;Z)`C+|zNg zE%?+HS{I!TU^TTECK{f~0|-CVts3ykl#XdNYZ~fs(S&qkkSbCk^ z-z%zIqM~pEQcVvl9`%7;I7x( zj?}Da3PMgr;wL!}#deAy=i?Kz)XqNu>tB4V7+%0i0qER$>moQ838ujhd&k$nxri-< zEBEd&Pd+!`s!XlNM`bt3QpC&*~tXGs9CsrM* zhZ8GRT#VL`M3=Na$jDVrGQL{a-_rih8wezfk$}v!R63KJMk;C!M6{iZ&yNzd;o(7iGF$U)BGRpMK;-Lll z)m9~=J(ZZLKzbKhvL9V0?W@MHGYth}@%Y+syekS52k68=yEEpxcborw^7mdC&9T6E zvNfENs4h+g_;#>QgIFi6Gb2Pb*-~5&^FpL}>KeYM2!8y9&B4*_WBf!ziZ{D3t98pk{ z?>DMzlA-3>l^w~g(T-FMuZU1s1LKP-^83F5(Hrvx7ipN{ zx;|LHy1u!#KBS`L7o!PLTsRU=g*&6EQ{i&+7B%EiIXj`ifKhZAq2j<~SV~?vll)Jo zj{1jH1gjPizh*dH$&}*yX~~qF;)&LoN(5kk^MQ5E#UpxXbLp=AeBd@k&u(Y?p$&E@ zyN7Ps9irn=`_>^zrfSXeY`A~bcdoV?+;M%PHsmqay6~8}kb(=U@THhasuX|)8amUh3JQ6TK7OUSIMBHu=A+?~FU)LDSoz z6TK7OUSIM~^iFuGXNOMoPI!BL$ver@j_=i1)$I7e?B7oBwgW0T$<39|h{y+LI)0Dc zKArU9fOp%eI<3B3`3zY;yL$|EU+ZIxI&JL!Ap{xVhewXA0kV*DU;RR6Y6T1GkkvDf4pg$!y`Z zxcUL>8u@L|ZrdSSovw1UpX>jN|tK{AvC=t@@o7)k_@IglT`q z7S@pl2^Wrt@~M-bX5~}Fm!eK-hg_&bZ|->X3ahQOYstQ=RhLnNROkLC|GykQJ&zi2 zFS+=R${P;Bs&8&Qz~12TVZJkcteX3hfu}gD?@V7QqdVQ(v@-1qB)brgLdO8sJUjPG zp5OgFvKs%Z0Qp(z}T1IKKJ__k9H@oQ*C!fn@ z*DYaE!zbTnWIeY z;L9U9T-u-)ezLHVIg+;?vL>)X|6J;f`KNYT+d>`5cvLM~=_uN!6$~sFutovrpQWx^HmydVKvA^+TVsj`FZfD-fl@RY@_oq%~6me%NoQQ-w z@DDCHRzQ!MtGrw`JSq(~@;>5WX@tEo@t5lF2Kavl_|0WDS7zd!1d5l!m#QTGRPy3a zB`=Oe^5Qfk|M!bw9$V=`k1>m|+5hxOEPCE@eyeC*Ec!G3@smFT)33?Y^lSWCHGTT< z!)p4|FkgLe-pgt!@f3YvlTFi?k1+*i+w^6UWTOPQaKqidmPo^Mhbhgi;<8f_2p7*0 znzL07qH7nr5^yEpO2CzXD*>wnMrtWs)XZ;nm%`?@v?7wpQn)tLrEqO_m%{W-vvnyv n1M|1s_Kh3VQur{2cCxw@*0+Y$-}52E8SbU9JxJ8NaWecDmW)Gi literal 0 HcmV?d00001 From 5d285f7868b0f4fc116a2e9dd1422d7db22759d8 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:01:05 +0200 Subject: [PATCH 12/43] add test for parser error --- .../data/parsers/parser_depth_average_rule.py | 12 +++---- .../parsers/test_parser_depth_average_rule.py | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 24da0234..df665b40 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -70,15 +70,13 @@ def _obtain_interface_suffix(self, layer_type: str, logger: ILogger): layer_type (str): z or sigma layers Returns: - interface_suffix: Suffix for z or sigma layers based on Delft3D + Suffix for z or sigma layers based on Delft3D defined suffixes """ if layer_type.lower() == 'z': - interface_suffix = INTERFACES_Z_SUFFIX + return INTERFACES_Z_SUFFIX elif layer_type.lower() == 'sigma': - interface_suffix = INTERFACES_SIGMA_SUFFIX + return INTERFACES_SIGMA_SUFFIX else: - logger.log_error(f"Layer_type '{layer_type}' is not recognized. Supported " - f"options are '{INTERFACES_Z_SUFFIX}' and " - f"'{INTERFACES_SIGMA_SUFFIX}'.") - return interface_suffix + raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " + f"Supported options are 'z' and 'sigma'.") diff --git a/tests/data/parsers/test_parser_depth_average_rule.py b/tests/data/parsers/test_parser_depth_average_rule.py index a704bb47..16c71ab4 100644 --- a/tests/data/parsers/test_parser_depth_average_rule.py +++ b/tests/data/parsers/test_parser_depth_average_rule.py @@ -9,6 +9,7 @@ """ from typing import Any, List + import pytest from mock import Mock @@ -38,6 +39,7 @@ def test_parse_dict_to_rule_data_logic(): { "name": "testname", "input_variable": "input", + "layer_type": "z", "output_variable": "output", } ) @@ -56,6 +58,7 @@ def test_parse_wrong_dict_to_rule_data_logic(): { "name": "testname", "output_variable": "output", + "layer_type": "sigma", } ) logger = Mock(ILogger) @@ -71,3 +74,32 @@ def test_parse_wrong_dict_to_rule_data_logic(): # Assert expected_message = "Missing element input_variable" assert exception_raised.args[0] == expected_message + +def test_parse_incorrect_layer_type_to_rule_data_logic(): + """Test if an incorrect layer_type not parsed""" + # Arrange + incorrect_layer_type = "UNKNOWN" + contents = dict( + { + "name": "testname", + "input_variable": "input", + "layer_type": incorrect_layer_type, + "output_variable": "output", + } + ) + logger = Mock(ILogger) + + # Act + data = ParserDepthAverageRule() + + with pytest.raises(NotImplementedError) as exc_info: + data.parse_dict(contents, logger) + + exception_raised = exc_info.value + + # Assert + #expected_message = f"Layer_type {incorrect_layer_type} is not recognized. \ Supported options are 'z' and 'sigma'." + + expected_message = f"Layer_type '{incorrect_layer_type}' is not recognized. \ +Supported options are 'z' and 'sigma'." + assert exception_raised.args[0] == expected_message From 56c1382eaacb787f083718b815d0d4e890827479 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:02:59 +0200 Subject: [PATCH 13/43] restore accidentally committed file --- .../test_data_access_layer_data/results.nc | Bin 6156 -> 6156 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/entities/test_data_access_layer_data/results.nc b/tests/data/entities/test_data_access_layer_data/results.nc index 07180be2125d9c6543a0a887e1be419d51da9fc7..c443c877abe03fdae015806969385647e3f2f714 100644 GIT binary patch delta 59 zcmeA%=rP!!#;RwilaZ2UYHO%xXryPvCe8o^j4Vtb3J6$AGILYkL)Z)-C$-+%9KyOo F2mo%P4I=;m delta 59 zcmeA%=rP!!#;RwelaZ2UYHO%xXrgD#Ce8o^j4Vtb3J6$AGILYkL)Z+LC-QP_4q@FP F1OR3b3%~#X From f1659c7a6001bfbfc0b07455f3ac0ad3eca572b5 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:04:23 +0200 Subject: [PATCH 14/43] line too long --- tests/business/entities/rules/test_depth_average_rule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/business/entities/rules/test_depth_average_rule.py b/tests/business/entities/rules/test_depth_average_rule.py index f009c6d4..27f5d824 100644 --- a/tests/business/entities/rules/test_depth_average_rule.py +++ b/tests/business/entities/rules/test_depth_average_rule.py @@ -126,7 +126,8 @@ def test_depth_average_rule( def test_dimension_error(): - """If the number of interfaces > number of layers + 1. Give an error, no calculation is possible""" + """If the number of interfaces > number of layers + 1. Give an error, no + calculation is possible""" logger = Mock(ILogger) rule = DepthAverageRule( name="test", From 8c0e65535150869a2df7288a7e98b4933da25f51 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:05:18 +0200 Subject: [PATCH 15/43] remove import --- decoimpact/business/entities/rules/depth_average_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 9fff30b3..f58f5e56 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -13,7 +13,6 @@ from typing import Dict import xarray as _xr -from regex import I from decoimpact.business.entities.rules.i_multi_array_based_rule import ( IMultiArrayBasedRule, From a357e09de23e6655593271dd052156aed60abf01 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:13:54 +0200 Subject: [PATCH 16/43] update documentation --- docs/manual/input.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/manual/input.md b/docs/manual/input.md index 88e12c8e..159f325a 100644 --- a/docs/manual/input.md +++ b/docs/manual/input.md @@ -610,10 +610,13 @@ FORMAT name: description: input_variable: + layer_type: output_variable: ``` -The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z-layers. The input file must include a variable called 'mesh2d_interface_z' over which the the input variable will be averaged. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. +The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z- or sigma-layers. The current implementation only supports input netCDF files generated by D-Hydro. The input file must include a variable called 'mesh2d_interface_z' or 'mesh2d_interface_sigma' over which the input variable will be averaged. Also two variables specifying the bedlevel and water level, 'mesh2d_flowelem_bl' and 'mesh2d_s1' are needed. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. + +Note: remapping the variables 'mesh2d_interface_z', 'mesh2d_interface_sigma', 'mesh2d_flowelem_bl and 'mesh2d_s1' in combination with the depth average rule is currently not supported. Combined z-sigma layers are also not supported. An explanation of how the depth rule works is shown in the example below. @@ -698,6 +701,7 @@ Below is an example of an input_file for the depth average rule: name: test depth average description: Test depth average input_variable: salinity + layer_type : sigma output_variable: average_salinity ``` From e36e15ea80de16c464315a95f1a21b5591086c2b Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Sat, 3 Aug 2024 01:22:38 +0200 Subject: [PATCH 17/43] fix unit tests --- tests/business/entities/rules/test_depth_average_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/business/entities/rules/test_depth_average_rule.py b/tests/business/entities/rules/test_depth_average_rule.py index 27f5d824..a0839892 100644 --- a/tests/business/entities/rules/test_depth_average_rule.py +++ b/tests/business/entities/rules/test_depth_average_rule.py @@ -95,7 +95,7 @@ def test_depth_average_rule( logger = Mock(ILogger) rule = DepthAverageRule( name="test", - input_variable_names=["foo"], + input_variable_names=["foo", "mesh2d_interface_z"], ) # Create dataset @@ -131,7 +131,7 @@ def test_dimension_error(): logger = Mock(ILogger) rule = DepthAverageRule( name="test", - input_variable_names=["foo"], + input_variable_names=["foo", "mesh2d_interface_z"], ) # Create dataset From ca9f8f022d98c699b777484589adaa0a2ff4c1c1 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 08:44:12 +0200 Subject: [PATCH 18/43] add layer type to depth average rule data --- decoimpact/business/entities/rules/depth_average_rule.py | 8 ++++++-- decoimpact/business/workflow/model_builder.py | 1 + decoimpact/data/api/i_depth_average_rule_data.py | 5 +++++ decoimpact/data/entities/depth_average_rule_data.py | 8 ++++++++ decoimpact/data/parsers/parser_depth_average_rule.py | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index f58f5e56..bd8d32a3 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -10,7 +10,7 @@ Classes: DepthAverageRule """ -from typing import Dict +from typing import Dict, List import xarray as _xr @@ -30,6 +30,10 @@ class DepthAverageRule(RuleBase, IMultiArrayBasedRule): """Implementation for the depth average rule""" + def __init__(self, name: str, input_variable_names: List[str], layer_type: str): + super().__init__(name, input_variable_names) + self._layer_type = layer_type + def execute( self, value_arrays: Dict[str, _xr.DataArray], logger: ILogger ) -> _xr.DataArray: @@ -46,7 +50,7 @@ def execute( # but the name of the key is given by the user, and is unknown here, so # just use the first value. variables = next(iter(value_arrays.values())) - + print('qq',self._layer_type) layer_type_suffix = self.determine_layer_type(logger) bed_level_values = self._extract_variable_based_on_suffix( diff --git a/decoimpact/business/workflow/model_builder.py b/decoimpact/business/workflow/model_builder.py index 0c039fdc..1eee980c 100644 --- a/decoimpact/business/workflow/model_builder.py +++ b/decoimpact/business/workflow/model_builder.py @@ -107,6 +107,7 @@ def _create_rule(rule_data: IRuleData) -> IRule: rule = DepthAverageRule( rule_data.name, rule_data.input_variables, + rule_data.layer_type, ) elif isinstance(rule_data, ILayerFilterRuleData): rule = LayerFilterRule( diff --git a/decoimpact/data/api/i_depth_average_rule_data.py b/decoimpact/data/api/i_depth_average_rule_data.py index 9272ba52..01eb9891 100644 --- a/decoimpact/data/api/i_depth_average_rule_data.py +++ b/decoimpact/data/api/i_depth_average_rule_data.py @@ -26,3 +26,8 @@ class IDepthAverageRuleData(IRuleData, ABC): @abstractmethod def input_variables(self) -> List[str]: """List with input variable name and standard depth name""" + + @property + @abstractmethod + def layer_type(self) -> str: + """Layer type of the model (z or sigma)""" diff --git a/decoimpact/data/entities/depth_average_rule_data.py b/decoimpact/data/entities/depth_average_rule_data.py index 99afa504..f3355022 100644 --- a/decoimpact/data/entities/depth_average_rule_data.py +++ b/decoimpact/data/entities/depth_average_rule_data.py @@ -13,6 +13,7 @@ """ from typing import List + from decoimpact.data.api.i_depth_average_rule_data import IDepthAverageRuleData from decoimpact.data.entities.rule_data import RuleData @@ -24,11 +25,18 @@ def __init__( self, name: str, input_variables: List[str], + layer_type: str, ): super().__init__(name) self._input_variables = input_variables + self._layer_type = layer_type @property def input_variables(self) -> List[str]: """List with input variables""" return self._input_variables + + @property + def layer_type(self) -> str: + """Layer type of the model (z or sigma)""" + return self._layer_type diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index df665b40..3b429dee 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -55,7 +55,7 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: output_variable_name: str = get_dict_element("output_variable", dictionary) description: str = get_dict_element("description", dictionary, False) or "" - rule_data = DepthAverageRuleData(name, input_variable_names) + rule_data = DepthAverageRuleData(name, input_variable_names, layer_type) rule_data.output_variable = output_variable_name rule_data.description = description From c3c04b7288ad57257e07c86bf9724e50c0cf3cb5 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 09:07:14 +0200 Subject: [PATCH 19/43] remove check layer function from depth average rule. Use layer type variable --- .../entities/rules/depth_average_rule.py | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index bd8d32a3..69fcd5b4 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -50,13 +50,12 @@ def execute( # but the name of the key is given by the user, and is unknown here, so # just use the first value. variables = next(iter(value_arrays.values())) - print('qq',self._layer_type) - layer_type_suffix = self.determine_layer_type(logger) + interface_suffix = self._get_layer_suffix(self._layer_type, logger) bed_level_values = self._extract_variable_based_on_suffix( value_arrays, BED_LEVEL_SUFFIX) depths_interfaces = self._extract_variable_based_on_suffix( - value_arrays, layer_type_suffix) + value_arrays, interface_suffix) water_level_values = self._extract_variable_based_on_suffix( value_arrays, WATER_LEVEL_SUFFIX) @@ -131,15 +130,15 @@ def _extract_variable_based_on_suffix( Returns: values (List[str]): Values based on prefix + suffix name """ - variable = [value_arrays[name] for name in value_arrays if suffix in name][0] - return variable + return [value_arrays[name] for name in value_arrays if suffix in name][0] - def determine_layer_type( + def _get_layer_suffix( self, + layer_type: str, logger: ILogger ): - """Determine whether the model is a sigma or Z layer model based on which - variable is present for the interfaces. Give error if both are found. + """Get the interface suffix depending on whether thei odel is a sigma or Z + layer model. Give error if the interface suffix cannot be determined. Args: value_array (DataArray): Values @@ -147,28 +146,10 @@ def determine_layer_type( Returns: layer_type (str): sigma or z """ - has_z_layers = self._check_if_layer_type_is_present("z") - has_sigma_layers = self._check_if_layer_type_is_present("sigma") - if has_sigma_layers and has_z_layers: - logger.log_error("Both Z and Sigma layers are present. This" - "combination is not supported for depth" - "averaging.") - if has_sigma_layers: + if layer_type.lower() == "sigma": return INTERFACES_SIGMA_SUFFIX - if has_z_layers: + elif layer_type.lower() == "z": return INTERFACES_Z_SUFFIX - - def _check_if_layer_type_is_present( - self, - layer_type: str, - ): - """Determine whether a layer type is present. - - Args: - layer_type (str): sigma or z - - Returns: - bool - """ - return any("interface_" + layer_type - in variable for variable in self._input_variable_names) + else: + logger.log_error(f"Layer type {layer_type} unknown. Interface " + "variable could not be determined.") From 4b4fec806703ca1f3c14a6bbc2ef375c9aa705be Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 10:55:49 +0200 Subject: [PATCH 20/43] fix unit tests --- tests/business/entities/rules/test_depth_average_rule.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/business/entities/rules/test_depth_average_rule.py b/tests/business/entities/rules/test_depth_average_rule.py index a0839892..eea985ba 100644 --- a/tests/business/entities/rules/test_depth_average_rule.py +++ b/tests/business/entities/rules/test_depth_average_rule.py @@ -24,7 +24,7 @@ def test_create_depth_average_rule_with_defaults(): """Test creating a depth average rule with defaults""" # Arrange & Act - rule = DepthAverageRule("test_rule_name", ["foo", "hello"]) + rule = DepthAverageRule("test_rule_name", ["foo", "hello"],"z") # Assert assert isinstance(rule, DepthAverageRule) @@ -32,6 +32,7 @@ def test_create_depth_average_rule_with_defaults(): assert rule.description == "" assert rule.input_variable_names == ["foo", "hello"] assert rule.output_variable_name == "output" + assert rule._layer_type == "z" def test_no_validate_error_with_correct_rule(): @@ -41,6 +42,7 @@ def test_no_validate_error_with_correct_rule(): rule = DepthAverageRule( "test_rule_name", ["foo", "hello"], + "z", ) # Assert @@ -96,6 +98,7 @@ def test_depth_average_rule( rule = DepthAverageRule( name="test", input_variable_names=["foo", "mesh2d_interface_z"], + layer_type="z", ) # Create dataset @@ -132,6 +135,7 @@ def test_dimension_error(): rule = DepthAverageRule( name="test", input_variable_names=["foo", "mesh2d_interface_z"], + layer_type="z", ) # Create dataset From c748e012950bf86ea48ba9a144ed18b54bd0118f Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 11:06:01 +0200 Subject: [PATCH 21/43] typo --- decoimpact/business/entities/rules/depth_average_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 69fcd5b4..413d7e1b 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -137,7 +137,7 @@ def _get_layer_suffix( layer_type: str, logger: ILogger ): - """Get the interface suffix depending on whether thei odel is a sigma or Z + """Get the interface suffix depending on whether the model is a sigma or z layer model. Give error if the interface suffix cannot be determined. Args: From 6371010ccd2341382811b7313b076a722953b3f2 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 11:54:25 +0200 Subject: [PATCH 22/43] move functions out of class --- .../entities/rules/depth_average_rule.py | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 413d7e1b..113289ab 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -50,13 +50,13 @@ def execute( # but the name of the key is given by the user, and is unknown here, so # just use the first value. variables = next(iter(value_arrays.values())) - interface_suffix = self._get_layer_suffix(self._layer_type, logger) + interface_suffix = _get_layer_suffix(self._layer_type, logger) - bed_level_values = self._extract_variable_based_on_suffix( + bed_level_values = _extract_variable_based_on_suffix( value_arrays, BED_LEVEL_SUFFIX) - depths_interfaces = self._extract_variable_based_on_suffix( + depths_interfaces = _extract_variable_based_on_suffix( value_arrays, interface_suffix) - water_level_values = self._extract_variable_based_on_suffix( + water_level_values = _extract_variable_based_on_suffix( value_arrays, WATER_LEVEL_SUFFIX) # Get the dimension names for the interfaces and for the layers @@ -115,41 +115,41 @@ def execute( dim=dim_layer_name ) - def _extract_variable_based_on_suffix( - self, - value_arrays: Dict[str, _xr.DataArray], - suffix: str - ) -> list: - """Extract the values from the XArray dataset based on the name - suffixes by matching the name, irrespective of the dummy name prefix. - Args: - value_array (DataArray): Values - suffix (str) : Suffix of the name - - Returns: - values (List[str]): Values based on prefix + suffix name - """ - return [value_arrays[name] for name in value_arrays if suffix in name][0] - - def _get_layer_suffix( - self, - layer_type: str, - logger: ILogger - ): - """Get the interface suffix depending on whether the model is a sigma or z - layer model. Give error if the interface suffix cannot be determined. - - Args: - value_array (DataArray): Values - - Returns: - layer_type (str): sigma or z - """ - if layer_type.lower() == "sigma": - return INTERFACES_SIGMA_SUFFIX - elif layer_type.lower() == "z": - return INTERFACES_Z_SUFFIX - else: - logger.log_error(f"Layer type {layer_type} unknown. Interface " - "variable could not be determined.") +def _extract_variable_based_on_suffix( + value_arrays: Dict[str, _xr.DataArray], + suffix: str + ) -> list: + """Extract the values from the XArray dataset based on the name + suffixes by matching the name, irrespective of the dummy name prefix. + + Args: + value_array (DataArray): Values + suffix (str) : Suffix of the name + + Returns: + values (List[str]): Values based on prefix + suffix name + """ + return [value_arrays[name] for name in value_arrays if suffix in name][0] + + +def _get_layer_suffix( + layer_type: str, + logger: ILogger + ): + """Get the interface suffix depending on whether the model is a sigma or z + layer model. Give error if the interface suffix cannot be determined. + + Args: + value_array (DataArray): Values + + Returns: + layer_type (str): sigma or z + """ + if layer_type.lower() == "sigma": + return INTERFACES_SIGMA_SUFFIX + elif layer_type.lower() == "z": + return INTERFACES_Z_SUFFIX + else: + logger.log_error(f"Layer type {layer_type} unknown. Interface " + "variable could not be determined.") From 72240b2d4e81447e83263f9acfe899c41cd0a465 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 11:55:49 +0200 Subject: [PATCH 23/43] add layer type to creation --- tests/data/entities/test_depth_average_rule_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data/entities/test_depth_average_rule_data.py b/tests/data/entities/test_depth_average_rule_data.py index 86c32ee9..08e7f7a7 100644 --- a/tests/data/entities/test_depth_average_rule_data.py +++ b/tests/data/entities/test_depth_average_rule_data.py @@ -18,9 +18,10 @@ def test_depth_average_rule_data_creation_logic(): to correctly initialize itself during creation""" # Act - data = DepthAverageRuleData("test_name", "input1") + data = DepthAverageRuleData("test_name", "input1", "z") # Assert assert isinstance(data, IRuleData) assert data.input_variables == "input1" + assert data.layer_type == "z" From fb84852d6c0640390e92b58a0af8c7d69c8395cb Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 14:26:24 +0200 Subject: [PATCH 24/43] remove import --- tests/business/entities/rules/test_depth_average_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/business/entities/rules/test_depth_average_rule.py b/tests/business/entities/rules/test_depth_average_rule.py index eea985ba..ef54e4aa 100644 --- a/tests/business/entities/rules/test_depth_average_rule.py +++ b/tests/business/entities/rules/test_depth_average_rule.py @@ -14,7 +14,6 @@ import numpy as _np import pytest import xarray as _xr -from tomlkit import value from decoimpact.business.entities.rules.depth_average_rule import DepthAverageRule from decoimpact.crosscutting.i_logger import ILogger From ac307944240527fa197abb937e103f80bba611da Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 15:21:30 +0200 Subject: [PATCH 25/43] remove unnecessary if's --- decoimpact/business/entities/rules/depth_average_rule.py | 7 +++---- decoimpact/data/parsers/parser_depth_average_rule.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 113289ab..8c4aa2eb 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -148,8 +148,7 @@ def _get_layer_suffix( """ if layer_type.lower() == "sigma": return INTERFACES_SIGMA_SUFFIX - elif layer_type.lower() == "z": + if layer_type.lower() == "z": return INTERFACES_Z_SUFFIX - else: - logger.log_error(f"Layer type {layer_type} unknown. Interface " - "variable could not be determined.") + logger.log_error(f"Layer type {layer_type} unknown. Interface " + "variable could not be determined.") diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 3b429dee..72154c53 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -75,8 +75,7 @@ def _obtain_interface_suffix(self, layer_type: str, logger: ILogger): """ if layer_type.lower() == 'z': return INTERFACES_Z_SUFFIX - elif layer_type.lower() == 'sigma': + if layer_type.lower() == 'sigma': return INTERFACES_SIGMA_SUFFIX - else: - raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " - f"Supported options are 'z' and 'sigma'.") + raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " + f"Supported options are 'z' and 'sigma'.") From e01fe927990729c418096e95b22f8cd9d33a42eb Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 15:23:16 +0200 Subject: [PATCH 26/43] remove imports. move function out of class --- decoimpact/business/utils/dataset_utils.py | 2 -- .../data/parsers/parser_depth_average_rule.py | 32 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/decoimpact/business/utils/dataset_utils.py b/decoimpact/business/utils/dataset_utils.py index b8af1b9a..5def5b03 100644 --- a/decoimpact/business/utils/dataset_utils.py +++ b/decoimpact/business/utils/dataset_utils.py @@ -6,11 +6,9 @@ # https://github.com/Deltares/D-EcoImpact/blob/main/LICENSE.md """Library for utility functions regarding an xarray dataset""" -from re import I from typing import List, Optional import xarray as _xr -from numpy import var import decoimpact.business.utils.list_utils as _lu from decoimpact.crosscutting.delft3d_specific_data import delft3d_specific_names diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 72154c53..ce1966d8 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -43,7 +43,7 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: """ name: str = get_dict_element("name", dictionary) layer_type: str = get_dict_element("layer_type", dictionary) - interface_suffix = self._obtain_interface_suffix(layer_type, logger) + interface_suffix = _obtain_interface_suffix(layer_type) input_variable_names: List[str] = [ get_dict_element("input_variable", dictionary), @@ -62,20 +62,20 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: return rule_data - def _obtain_interface_suffix(self, layer_type: str, logger: ILogger): - """Obtain the interface variable based on the layer_type specified. - Give an error if layer_type is not recognised +def _obtain_interface_suffix(layer_type: str): + """Obtain the interface variable based on the layer_type specified. + Give an error if layer_type is not recognised - Args: - layer_type (str): z or sigma layers + Args: + layer_type (str): z or sigma layers - Returns: - Suffix for z or sigma layers based on Delft3D - defined suffixes - """ - if layer_type.lower() == 'z': - return INTERFACES_Z_SUFFIX - if layer_type.lower() == 'sigma': - return INTERFACES_SIGMA_SUFFIX - raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " - f"Supported options are 'z' and 'sigma'.") + Returns: + Suffix for z or sigma layers based on Delft3D + defined suffixes + """ + if layer_type.lower() == 'z': + return INTERFACES_Z_SUFFIX + if layer_type.lower() == 'sigma': + return INTERFACES_SIGMA_SUFFIX + raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " + f"Supported options are 'z' and 'sigma'.") From 3e67a26d7fef96e3a4d122bee51e1548ed93b876 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 15:25:32 +0200 Subject: [PATCH 27/43] fix flake --- decoimpact/business/entities/rules/depth_average_rule.py | 2 +- decoimpact/data/parsers/parser_depth_average_rule.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 8c4aa2eb..c9cd4ffd 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -151,4 +151,4 @@ def _get_layer_suffix( if layer_type.lower() == "z": return INTERFACES_Z_SUFFIX logger.log_error(f"Layer type {layer_type} unknown. Interface " - "variable could not be determined.") + f"variable could not be determined.") diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index ce1966d8..1d6e34c5 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -78,4 +78,4 @@ def _obtain_interface_suffix(layer_type: str): if layer_type.lower() == 'sigma': return INTERFACES_SIGMA_SUFFIX raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " - f"Supported options are 'z' and 'sigma'.") + f"Supported options are 'z' and 'sigma'.") From bde07c8496f3bb36be06dbfe9c16af045dd42225 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 15:26:52 +0200 Subject: [PATCH 28/43] lsit -> List --- decoimpact/business/entities/rules/depth_average_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index c9cd4ffd..84c0e5fe 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -119,7 +119,7 @@ def execute( def _extract_variable_based_on_suffix( value_arrays: Dict[str, _xr.DataArray], suffix: str - ) -> list: + ) -> List: """Extract the values from the XArray dataset based on the name suffixes by matching the name, irrespective of the dummy name prefix. From d5a47e35a4079be5c0712707a1309b33c14746a6 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Mon, 5 Aug 2024 15:28:32 +0200 Subject: [PATCH 29/43] final flake --- decoimpact/data/parsers/parser_depth_average_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 1d6e34c5..8257d7ed 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -62,6 +62,7 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: return rule_data + def _obtain_interface_suffix(layer_type: str): """Obtain the interface variable based on the layer_type specified. Give an error if layer_type is not recognised From 7b21ffb235f32347acbf710d02a5cccbc614b39e Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 11:38:35 +0200 Subject: [PATCH 30/43] fix the error when a variable name is remapped --- decoimpact/business/entities/rule_based_model.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 294361dd..389eb472 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -198,10 +198,17 @@ def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: rule_inputs = self._get_direct_rule_inputs(rule_names) + for dataset in self._input_datasets: + dummy_var_name = _du.get_dummy_variable_in_ugrid(dataset) + # check for missing rule inputs for rule_name, rule_input in rule_inputs.items(): needed_rule_inputs = _lu.remove_duplicates_from_list(rule_input) rule_input_vars = input_vars + list(mappings.values()) + needed_rule_inputs = _du.extend_to_full_name( + needed_rule_inputs, + dummy_var_name + ) missing_rule_inputs = _lu.items_not_in(needed_rule_inputs, rule_input_vars) if len(missing_rule_inputs) > 0: logger.log_error( From 5ce35eb460c225ae1edfd2e2771bd95d84a789a8 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:21:45 +0200 Subject: [PATCH 31/43] add check on illegal suffixes in remapping --- .../business/entities/rule_based_model.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 389eb472..1b2138d7 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -21,6 +21,13 @@ from decoimpact.business.entities.i_model import IModel, ModelStatus from decoimpact.business.entities.rule_processor import RuleProcessor from decoimpact.business.entities.rules.i_rule import IRule +from decoimpact.crosscutting.delft3d_specific_data import ( + BED_LEVEL_SUFFIX, + INTERFACES_SIGMA_SUFFIX, + INTERFACES_Z_SUFFIX, + WATER_LEVEL_SUFFIX, + delft3d_specific_names, +) from decoimpact.crosscutting.i_logger import ILogger @@ -169,6 +176,7 @@ def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: for ds in self._input_datasets ] ) + self._check_keys_with_suffixes(mappings, delft3d_specific_names) valid = True @@ -246,3 +254,19 @@ def _extend_names(self, dummy_variable_name: str): rule.input_variable_names, dummy_variable_name ) + + def _check_keys_with_suffixes(self, dictionary, suffixes): + """ + Checks if any key in the dictionary ends with provided suffixes. + Raise eror if that occurs. + + Args: + dictionary (dict) : The dictionary to check. + suffixes (List[str]): List of suffixes to check against. + + """ + for key in dictionary: + if any(key.endswith(suffix) for suffix in suffixes): + raise ValueError(f"Remapping variables ending with" + f" {delft3d_specific_names} is not" + f" allowed.") From 537d0a563e435478ee394c020b62fdd989aa77d9 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:21:51 +0200 Subject: [PATCH 32/43] fix test --- tests/business/entities/test_rule_based_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/business/entities/test_rule_based_model.py b/tests/business/entities/test_rule_based_model.py index ab99e102..080c84f6 100644 --- a/tests/business/entities/test_rule_based_model.py +++ b/tests/business/entities/test_rule_based_model.py @@ -72,6 +72,7 @@ def test_validation_of_rule_based_model(): logger = Mock(ILogger) dataset["test"] = _xr.DataArray([32, 94, 9]) + dataset["test"].attrs = {"cf_role": "mesh_topology"} rule.input_variable_names = ["input"] rule.output_variable_name = "output" From 931dc71d6e1456b8e39f53ec8457fee9fb387fe2 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:25:11 +0200 Subject: [PATCH 33/43] remove unneeded imports --- decoimpact/business/entities/rule_based_model.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 1b2138d7..05d25db6 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -21,13 +21,7 @@ from decoimpact.business.entities.i_model import IModel, ModelStatus from decoimpact.business.entities.rule_processor import RuleProcessor from decoimpact.business.entities.rules.i_rule import IRule -from decoimpact.crosscutting.delft3d_specific_data import ( - BED_LEVEL_SUFFIX, - INTERFACES_SIGMA_SUFFIX, - INTERFACES_Z_SUFFIX, - WATER_LEVEL_SUFFIX, - delft3d_specific_names, -) +from decoimpact.crosscutting.delft3d_specific_data import delft3d_specific_names from decoimpact.crosscutting.i_logger import ILogger From 2c877ab7c7deab195139fbd1146d6e119c79f071 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:27:15 +0200 Subject: [PATCH 34/43] disable pylint warning --- decoimpact/business/entities/rules/depth_average_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 84c0e5fe..42b800de 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -34,6 +34,7 @@ def __init__(self, name: str, input_variable_names: List[str], layer_type: str): super().__init__(name, input_variable_names) self._layer_type = layer_type + # pylint: disable=too-many-arguments def execute( self, value_arrays: Dict[str, _xr.DataArray], logger: ILogger ) -> _xr.DataArray: From 2116bfd1651e9dc47b7639569637ba5e923c8147 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:28:25 +0200 Subject: [PATCH 35/43] extend error message --- decoimpact/business/entities/rules/depth_average_rule.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 42b800de..73b8468a 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -151,5 +151,6 @@ def _get_layer_suffix( return INTERFACES_SIGMA_SUFFIX if layer_type.lower() == "z": return INTERFACES_Z_SUFFIX - logger.log_error(f"Layer type {layer_type} unknown. Interface " - f"variable could not be determined.") + logger.log_error(f"Layer type {layer_type} unknown. Allowed layer " + "type: z or sigma. Interface " + "variable could not be determined.") From 20ac68d9b2b5776b36461b8627478bbcccbb1f65 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:36:20 +0200 Subject: [PATCH 36/43] disable pylint warning --- .../business/entities/rule_based_model.py | 1 + .../test_data_access_layer_data/results.nc | Bin 6156 -> 6156 bytes 2 files changed, 1 insertion(+) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 05d25db6..a8a75147 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -154,6 +154,7 @@ def _make_output_variables_list(self) -> list: return _lu.remove_duplicates_from_list(all_vars) + #pylint: disable=R0194 def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: """Checks if the provided mappings are valid. diff --git a/tests/data/entities/test_data_access_layer_data/results.nc b/tests/data/entities/test_data_access_layer_data/results.nc index c443c877abe03fdae015806969385647e3f2f714..07180be2125d9c6543a0a887e1be419d51da9fc7 100644 GIT binary patch delta 59 zcmeA%=rP!!#;RwelaZ2UYHO%xXrgD#Ce8o^j4Vtb3J6$AGILYkL)Z+LC-QP_4q@FP F1OR3b3%~#X delta 59 zcmeA%=rP!!#;RwilaZ2UYHO%xXryPvCe8o^j4Vtb3J6$AGILYkL)Z)-C$-+%9KyOo F2mo%P4I=;m From 2ff39d8a953fce474cce7ab0f550ee71b4b96cfb Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:37:58 +0200 Subject: [PATCH 37/43] Revert "disable pylint warning" This reverts commit 20ac68d9b2b5776b36461b8627478bbcccbb1f65. --- .../business/entities/rule_based_model.py | 1 - .../test_data_access_layer_data/results.nc | Bin 6156 -> 6156 bytes 2 files changed, 1 deletion(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index a8a75147..05d25db6 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -154,7 +154,6 @@ def _make_output_variables_list(self) -> list: return _lu.remove_duplicates_from_list(all_vars) - #pylint: disable=R0194 def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: """Checks if the provided mappings are valid. diff --git a/tests/data/entities/test_data_access_layer_data/results.nc b/tests/data/entities/test_data_access_layer_data/results.nc index 07180be2125d9c6543a0a887e1be419d51da9fc7..c443c877abe03fdae015806969385647e3f2f714 100644 GIT binary patch delta 59 zcmeA%=rP!!#;RwilaZ2UYHO%xXryPvCe8o^j4Vtb3J6$AGILYkL)Z)-C$-+%9KyOo F2mo%P4I=;m delta 59 zcmeA%=rP!!#;RwelaZ2UYHO%xXrgD#Ce8o^j4Vtb3J6$AGILYkL)Z+LC-QP_4q@FP F1OR3b3%~#X From 755967f10f7bf03d19990afe80ac9a17440a2b6b Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:38:42 +0200 Subject: [PATCH 38/43] remove comment --- tests/data/parsers/test_parser_depth_average_rule.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/data/parsers/test_parser_depth_average_rule.py b/tests/data/parsers/test_parser_depth_average_rule.py index 16c71ab4..44e50465 100644 --- a/tests/data/parsers/test_parser_depth_average_rule.py +++ b/tests/data/parsers/test_parser_depth_average_rule.py @@ -98,8 +98,6 @@ def test_parse_incorrect_layer_type_to_rule_data_logic(): exception_raised = exc_info.value # Assert - #expected_message = f"Layer_type {incorrect_layer_type} is not recognized. \ Supported options are 'z' and 'sigma'." - expected_message = f"Layer_type '{incorrect_layer_type}' is not recognized. \ Supported options are 'z' and 'sigma'." assert exception_raised.args[0] == expected_message From 8b7884715afccaaf65c85ad4d88725297f42142d Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:42:51 +0200 Subject: [PATCH 39/43] add return of unknown suffix --- decoimpact/business/entities/rules/depth_average_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index 73b8468a..dbd36dba 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -154,3 +154,4 @@ def _get_layer_suffix( logger.log_error(f"Layer type {layer_type} unknown. Allowed layer " "type: z or sigma. Interface " "variable could not be determined.") + return "_unknown" From b1dad4fb895edc571762ef461575955bed17b53b Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:48:13 +0200 Subject: [PATCH 40/43] disable pylint warnings --- decoimpact/business/entities/rule_based_model.py | 1 + decoimpact/business/entities/rules/depth_average_rule.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 05d25db6..4b867f7a 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -154,6 +154,7 @@ def _make_output_variables_list(self) -> list: return _lu.remove_duplicates_from_list(all_vars) + #pylint disable=too-many-locals def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: """Checks if the provided mappings are valid. diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index dbd36dba..8119429f 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -34,7 +34,7 @@ def __init__(self, name: str, input_variable_names: List[str], layer_type: str): super().__init__(name, input_variable_names) self._layer_type = layer_type - # pylint: disable=too-many-arguments + # pylint: disable=too-many-locals def execute( self, value_arrays: Dict[str, _xr.DataArray], logger: ILogger ) -> _xr.DataArray: From be1f265cc8fa453a29b49dfde99083d93a77490e Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Tue, 6 Aug 2024 13:50:31 +0200 Subject: [PATCH 41/43] correct ignore message --- decoimpact/business/entities/rule_based_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 4b867f7a..5b2bdac1 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -154,7 +154,7 @@ def _make_output_variables_list(self) -> list: return _lu.remove_duplicates_from_list(all_vars) - #pylint disable=too-many-locals + # pylint: disable=too-many-locals def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: """Checks if the provided mappings are valid. From bb947dcaf9c8e57dc96f0b0263e1b3c84b662e14 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Wed, 7 Aug 2024 10:14:15 +0200 Subject: [PATCH 42/43] update depth average rule documentation --- docs/manual/rules/depth_average_rule.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/manual/rules/depth_average_rule.md b/docs/manual/rules/depth_average_rule.md index eafec297..2781f763 100644 --- a/docs/manual/rules/depth_average_rule.md +++ b/docs/manual/rules/depth_average_rule.md @@ -1,4 +1,3 @@ - ### Depth average rule ``` @@ -7,14 +6,17 @@ FORMAT name: description: input_variable: + layer_type: output_variable: ``` -The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z-layers. The input file must include a variable called 'mesh2d_interface_z' over which the the input variable will be averaged. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. +The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z- or sigma-layers. The current implementation only supports input netCDF files generated by D-Hydro. The input file must include a variable called 'mesh2d_interface_z' or 'mesh2d_interface_sigma' over which the input variable will be averaged. Also two variables specifying the bedlevel and water level, 'mesh2d_flowelem_bl' and 'mesh2d_s1' are needed. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. + +Note: remapping the variables 'mesh2d_interface_z', 'mesh2d_interface_sigma', 'mesh2d_flowelem_bl and 'mesh2d_s1' in combination with the depth average rule is currently not supported. Combined z-sigma layers are also not supported. An explanation of how the depth rule works is shown in the example below. -![Example depth average rule](../../assets/images/3_depth_average.png "An example of a simplified grid with Z-layers. This model has 6 faces, 4 layers and 2 timesteps.") +![Example depth average rule](../assets/images/3_depth_average.png "An example of a simplified grid with Z-layers. This model has 6 faces, 4 layers and 2 timesteps.") The image shows a simplified model with the following dimensions: - mesh2d_nFaces = 6 (number of faces) @@ -95,6 +97,6 @@ Below is an example of an input_file for the depth average rule: name: test depth average description: Test depth average input_variable: salinity + layer_type : sigma output_variable: average_salinity -``` - +``` \ No newline at end of file From d4fc2ad50cb6d20965a4dd021a9939d8f0044fc6 Mon Sep 17 00:00:00 2001 From: mKlapwijk Date: Wed, 7 Aug 2024 10:16:48 +0200 Subject: [PATCH 43/43] add note on remapping several variables --- docs/manual/input.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/manual/input.md b/docs/manual/input.md index 2458347a..ade420d0 100644 --- a/docs/manual/input.md +++ b/docs/manual/input.md @@ -19,6 +19,8 @@ output-data: In the input data the variables that are present in the input data provided through “filename” are selected for use. It is possible to filter the input data by providing a start date or end date (format: "dd-mm-yyyy"); this is optional. The variables that are used can be selected under “variable_mapping”. Here you are also able to rename variables as the name used for storage is often cryptic. +Note: remapping the variables 'mesh2d_interface_z', 'mesh2d_interface_sigma', 'mesh2d_flowelem_bl and 'mesh2d_s1' is currently not supported to enable depth-averaging. This will be remedied in a future version. For now a work-around is to use a multiply rule with a multiplication factor of 1. This will in effect create a new variable, to which the remapped name can be assigned as output. + At output data the location where the output file needs to be written can be provided through “filename”. In this output file only variables that have been used from the input data and variables that have been created in the model are stored. It is possible to reduce the file size with the optional parameter "save_only_variables", which can take the name of one or several variables. The model needs at least one rule under “rules” to execute.