From 0f72e9ab93d49485a89b41e437ed4cad6fd7dfe6 Mon Sep 17 00:00:00 2001 From: Maria Date: Sat, 10 Apr 2021 18:53:14 +0200 Subject: [PATCH 01/17] Sample if true function added --- pysd/py_backend/functions.py | 87 ++++++++++++++++++++++++ pysd/py_backend/vensim/vensim2py.py | 15 ++++ tests/integration_test_vensim_pathway.py | 2 - 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index 5e326638..77ffdfb1 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -1186,6 +1186,93 @@ def random_uniform(m, x, s): return np.random.uniform(m, x) +# dictionary to store the values from SAMPLE IF TRUE function +saved_value = {} + +def make_da(rows, cols, initial_value): + """ + Returns a DataArray with the coordinates + of the rows and cols. + DataArray values are initialized with the initial_value. + It is used in SAMPLE IF TRUE function, to create the proper dimension saved value. + + Parameters + ---------- + rows: float or xarray.DataArray + Represents the row dimension of the new DataArray + cols: xarray.DataArray + Represents the col dimension of the new DataArray + initial_value: float or xarray.DataArray + Include the values to initialize the new DataArray + + Returns + ------- + A new DataArray with proper rows and cols coordinates, + initialized with initial_value + + """ + if(isinstance(initial_value,xr.DataArray)): + array = np.array([[initial_value.data[e] for i in range(0, len(rows.values))] for e in range(0,len(cols.values))]) + elif(isinstance(rows, xr.DataArray)): + array = np.array([[initial_value for i in range(0, len(rows.values))] for i in range(0,len(cols.values))]) + else: + array = np.array([initial_value for i in range(0,len(cols.values))]) + + coords = {dim: cols.coords[dim] for dim in cols.dims} + dims = cols.dims + if(isinstance(rows, xr.DataArray)): + coords.update({dim: rows.coords[dim] for dim in rows.dims}) + dims += rows.dims + return xr.DataArray(data=array, coords=coords, dims=dims) + +def sample_if_true(time, condition, actual_value, initial_value, var_name): + """ + Implements Vensim's SAMPLE IF TRUE function. + + Parameters + ---------- + condition: bool or xarray.DataArray + actual_value: float or xarray.DataArray + Value to return when condition is true. + initial_value: float or xarray.DataArray + Value to return when condition is false. + var_name: str + Represents the SAMPLE IF TRUE function in the whole model. + + Returns + ------- + float or xarray.DataArray + Actual_value when condition is true and saved this value + in saved_value dictionary. + Returns the last saved value when condicion is false. + Saved value is initialized with initial_value in the first step of simulation. + """ + global saved_value + t = time() + + if(t==0): + if(not(isinstance(condition,xr.DataArray))): + saved_value[var_name] = initial_value + else: + saved_value[var_name] = make_da(actual_value, condition, initial_value) + + if isinstance(condition, xr.DataArray): + if condition.all(): + for i in range(0,len(saved_value[var_name].values)): + saved_value[var_name].values[i]=actual_value + return saved_value[var_name] + elif not condition.any(): + return saved_value[var_name] + + for i in range(0, len(condition)): + if condition.values[i]: + saved_value[var_name][i]=actual_value + return xr.where(condition, actual_value, saved_value[var_name]) + + if condition: + saved_value[var_name] = actual_value + + return saved_value[var_name] def incomplete(*args): warnings.warn( diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index adf696fb..102ac6ad 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -433,6 +433,17 @@ def parse_units(units_str): ], "module": "functions" }, + "sample if true": { + "name": "sample_if_true", + "parameters": [ + {"name": 'time', "type": 'time'}, + {"name": 'condition'}, + {"name": 'val_if_true'}, + {"name": 'val_if_false'}, + {"name": 'var_name'} + ], + "module": "functions" + }, "step": { "name": "step", "parameters": [ @@ -892,6 +903,10 @@ def visit_call(self, n, vc): if self.apply_dim and function_name in vectorial_funcs: arguments += ["dim="+str(tuple(self.apply_dim))] self.apply_dim = set() + + # add name of variable to sample if true function arguments + if(isinstance(functions[function_name],dict) and functions[function_name]['name'] == "sample_if_true"): + arguments.append(element['py_name']) return builder.build_function_call(functions[function_name], arguments) diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py index 5d9e3f54..72f2e4c9 100644 --- a/tests/integration_test_vensim_pathway.py +++ b/tests/integration_test_vensim_pathway.py @@ -300,9 +300,7 @@ def test_rounding(self): output, canon = runner('test-models/tests/rounding/test_rounding.mdl') assert_frames_close(output, canon, rtol=rtol) - @unittest.skip('working on it #217') def test_sample_if_true(self): - # issue https://github.com/JamesPHoughton/pysd/issues/217 from.test_utils import runner, assert_frames_close output, canon = runner('test-models/tests/sample_if_true/test_sample_if_true.mdl') assert_frames_close(output, canon, rtol=rtol) From 3c386ff1fe3190c607780c7e71baa288d27e5457 Mon Sep 17 00:00:00 2001 From: Maria Date: Sun, 11 Apr 2021 11:11:56 +0200 Subject: [PATCH 02/17] Added subscript sequence support --- pysd/py_backend/vensim/vensim2py.py | 58 +++++++++++++++++++++++- tests/integration_test_vensim_pathway.py | 5 ++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 102ac6ad..e8c7709a 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -256,7 +256,7 @@ def get_equation_components(equation_str, root_path=None): component_structure_grammar = _include_common_grammar(r""" entry = component / data_definition / test_definition / subscript_definition / lookup_definition component = name _ subscriptlist? _ "=" "="? _ expression - subscript_definition = name _ ":" _ (imported_subscript / literal_subscript) + subscript_definition = name _ ":" _ (imported_subscript / literal_subscript / subscript_sequence) data_definition = name _ subscriptlist? _ keyword? _ ":=" _ expression lookup_definition = name _ subscriptlist? &"(" _ expression # uses lookahead assertion to capture whole group test_definition = name _ subscriptlist? _ &keyword _ expression @@ -264,10 +264,14 @@ def get_equation_components(equation_str, root_path=None): name = basic_id / escape_group literal_subscript = subscript _ ("," _ subscript _)* imported_subscript = func _ "(" _ (string _ ","? _)* ")" + subscript_sequence = "(" _ sequence_id _ "-" _ sequence_id _ ")" subscriptlist = '[' _ subscript _ ("," _ subscript _)* _ ']' expression = ~r".*" # expression could be anything, at this point. keyword = ":" _ basic_id _ ":" + sequence_id = id_common+ numbers+ + id_common = ~r"[A-Z]"i + numbers = ~r"[0-9]" subscript = basic_id / escape_group func = basic_id string = "\'" ( "\\\'" / ~r"[^\']"IU )* "\'" @@ -313,6 +317,14 @@ def visit_imported_subscript(self, n, vc): args_str = vc[4] # todo: make this less fragile? self.subscripts += get_external_data(f_str, args_str, root_path) + def visit_subscript_sequence(self, n, vc): + subscript_start = vc[2].strip() + subscript_end = vc[6].strip() + substring, start, end = get_subscript_sequence(subscript_start, subscript_end) + for i in range(start, end+1): + s = substring + str(i) + self.subscripts.append(s.strip()) + def visit_name(self, n, vc): (name,) = vc self.real_name = name.strip() @@ -350,6 +362,50 @@ def get_external_data(func_str, args_str, root_path): return f(*args, root=root_path).subscript +def get_subscript_sequence(subs_start, subs_end): + """ + With the first and the last subscript of a subscript sequence, + gets the common string of both and the starting and ending + number of the subscript sequence + + Parameters + ---------- + subs_start: str + Represents the first subscript value in the subscript sequence + subs_end: str + Represents the last subscript value in the subscript sequence + + Returns + ------- + common: str + Common string of both subscripts + num_start: int + Sequence start number + num_end: int + Sequence end number + + Examples + -------- + >>> get_subscript_sequence('Layer1', 'Layer5') + ('Layer', 1, 5) + >>> get_subscript_sequence('sub15', 'sub30') + ('sub', 15, 30) + """ + subs_start_l = list(subs_start) + subs_end_l = list(subs_end) + common = "" + for i in range(len(subs_start)): + if(subs_start_l[i] == subs_end_l[i] and not(subs_start_l[i].isdigit())): + common = common + subs_start_l[i] + else: break + num_start = subs_start[i:] + num_end = subs_end[i:] + if(not(num_start.isdigit()) or not(num_end.isdigit()) ): raise ValueError("Format of subscript sequence is not correct\n") + num_start = int(num_start) + num_end = int(num_end) + if(num_start>num_end): raise ValueError("The number of the first subscript must be lower than the second subscript number in a subscript sequence\n") + + return common, num_start, num_end def parse_units(units_str): """ diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py index 72f2e4c9..fd1d067a 100644 --- a/tests/integration_test_vensim_pathway.py +++ b/tests/integration_test_vensim_pathway.py @@ -406,6 +406,11 @@ def test_subscript_selection(self): output, canon = runner('test-models/tests/subscript_selection/subscript_selection.mdl') assert_frames_close(output, canon, rtol=rtol) + def test_subscript_sequence(self): + from test_utils import runner, assert_frames_close + output, canon = runner('test-models/tests/subscript_sequence/test_subscript_sequence.mdl') + assert_frames_close(output, canon, rtol=rtol) + def test_subscript_subranges(self): from.test_utils import runner, assert_frames_close output, canon = runner('test-models/tests/subscript_subranges/test_subscript_subrange.mdl') From 532a089dd227b79c51c647c373a9049b33e79047 Mon Sep 17 00:00:00 2001 From: Maria Date: Sat, 17 Apr 2021 11:13:48 +0200 Subject: [PATCH 03/17] Updated name of subscript sequence to numeric range, and added a grammatical rule --- pysd/py_backend/vensim/vensim2py.py | 39 +++++++++++++----------- tests/integration_test_vensim_pathway.py | 4 +-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index e8c7709a..c3ed5dd5 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -256,7 +256,7 @@ def get_equation_components(equation_str, root_path=None): component_structure_grammar = _include_common_grammar(r""" entry = component / data_definition / test_definition / subscript_definition / lookup_definition component = name _ subscriptlist? _ "=" "="? _ expression - subscript_definition = name _ ":" _ (imported_subscript / literal_subscript / subscript_sequence) + subscript_definition = name _ ":" _ (imported_subscript / literal_subscript / numeric_range) data_definition = name _ subscriptlist? _ keyword? _ ":=" _ expression lookup_definition = name _ subscriptlist? &"(" _ expression # uses lookahead assertion to capture whole group test_definition = name _ subscriptlist? _ &keyword _ expression @@ -264,7 +264,9 @@ def get_equation_components(equation_str, root_path=None): name = basic_id / escape_group literal_subscript = subscript _ ("," _ subscript _)* imported_subscript = func _ "(" _ (string _ ","? _)* ")" - subscript_sequence = "(" _ sequence_id _ "-" _ sequence_id _ ")" + numeric_range = _ (range / value) _ ("," _ (range / value) _)* + value = _ sequence_id _ + range = "(" _ sequence_id _ "-" _ sequence_id _ ")" subscriptlist = '[' _ subscript _ ("," _ subscript _)* _ ']' expression = ~r".*" # expression could be anything, at this point. keyword = ":" _ basic_id _ ":" @@ -317,14 +319,17 @@ def visit_imported_subscript(self, n, vc): args_str = vc[4] # todo: make this less fragile? self.subscripts += get_external_data(f_str, args_str, root_path) - def visit_subscript_sequence(self, n, vc): - subscript_start = vc[2].strip() - subscript_end = vc[6].strip() - substring, start, end = get_subscript_sequence(subscript_start, subscript_end) + def visit_range(self, n, vc): + subs_start = vc[2].strip() + subs_end = vc[6].strip() + self.sequence, start, end = get_subscript_number_range(subs_start, subs_end) for i in range(start, end+1): - s = substring + str(i) + s = self.sequence + str(i) self.subscripts.append(s.strip()) + def visit_value(self, n, vc): + self.subscripts.append(vc[1]) + def visit_name(self, n, vc): (name,) = vc self.real_name = name.strip() @@ -362,18 +367,18 @@ def get_external_data(func_str, args_str, root_path): return f(*args, root=root_path).subscript -def get_subscript_sequence(subs_start, subs_end): +def get_subscript_number_range(subs_start, subs_end): """ - With the first and the last subscript of a subscript sequence, - gets the common string of both and the starting and ending - number of the subscript sequence + With the first and the last subscript values of a subscript + numeric range, gets the common string of both and the + starting and ending number of the numeric range Parameters ---------- subs_start: str - Represents the first subscript value in the subscript sequence + Represents the first subscript value in the numeric range subs_end: str - Represents the last subscript value in the subscript sequence + Represents the last subscript value in the numeric range Returns ------- @@ -386,9 +391,9 @@ def get_subscript_sequence(subs_start, subs_end): Examples -------- - >>> get_subscript_sequence('Layer1', 'Layer5') + >>> get_subscript_number_range('Layer1', 'Layer5') ('Layer', 1, 5) - >>> get_subscript_sequence('sub15', 'sub30') + >>> get_subscript_number_range('sub15', 'sub30') ('sub', 15, 30) """ subs_start_l = list(subs_start) @@ -400,10 +405,10 @@ def get_subscript_sequence(subs_start, subs_end): else: break num_start = subs_start[i:] num_end = subs_end[i:] - if(not(num_start.isdigit()) or not(num_end.isdigit()) ): raise ValueError("Format of subscript sequence is not correct\n") + if(not(num_start.isdigit()) or not(num_end.isdigit()) ): raise ValueError("Format of subscript numeric range is not correct\n") num_start = int(num_start) num_end = int(num_end) - if(num_start>num_end): raise ValueError("The number of the first subscript must be lower than the second subscript number in a subscript sequence\n") + if(num_start>num_end): raise ValueError("The number of the first subscript value must be lower than the second subscript value in a subscript numeric range\n") return common, num_start, num_end diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py index fd1d067a..65636807 100644 --- a/tests/integration_test_vensim_pathway.py +++ b/tests/integration_test_vensim_pathway.py @@ -406,9 +406,9 @@ def test_subscript_selection(self): output, canon = runner('test-models/tests/subscript_selection/subscript_selection.mdl') assert_frames_close(output, canon, rtol=rtol) - def test_subscript_sequence(self): + def test_subscript_numeric_range(self): from test_utils import runner, assert_frames_close - output, canon = runner('test-models/tests/subscript_sequence/test_subscript_sequence.mdl') + output, canon = runner('test-models/tests/subscript_numeric_range/test_subscript_numeric_range.mdl') assert_frames_close(output, canon, rtol=rtol) def test_subscript_subranges(self): From 0baf3dcb98b8cbb3d450f1a8cfc57ec5b2e226ca Mon Sep 17 00:00:00 2001 From: Maria Date: Sat, 17 Apr 2021 11:16:51 +0200 Subject: [PATCH 04/17] Changed the way to split arguments in visit_call of expression_grammar --- pysd/py_backend/vensim/vensim2py.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index c3ed5dd5..65b5907b 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -958,7 +958,10 @@ def visit_call(self, n, vc): # remove dimensions info (produced by !) function_name = vc[0].lower() - arguments = [e.strip() for e in vc[4].split(",")] + arguments = [] + while len(','.join(arguments)) < len(vc[4]): + arguments.append(self.args.pop()) + arguments = [arguments[-1]] + arguments[:-1] # add dimensions as last argument if self.apply_dim and function_name in vectorial_funcs: From ce36612b8eaeff8f812a8d3d6f92e8f0d736e649 Mon Sep 17 00:00:00 2001 From: Maria Date: Sat, 17 Apr 2021 11:42:32 +0200 Subject: [PATCH 05/17] Updated import in test_subscript_numeric_range --- tests/integration_test_vensim_pathway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py index 65636807..479b896a 100644 --- a/tests/integration_test_vensim_pathway.py +++ b/tests/integration_test_vensim_pathway.py @@ -407,7 +407,7 @@ def test_subscript_selection(self): assert_frames_close(output, canon, rtol=rtol) def test_subscript_numeric_range(self): - from test_utils import runner, assert_frames_close + from .test_utils import runner, assert_frames_close output, canon = runner('test-models/tests/subscript_numeric_range/test_subscript_numeric_range.mdl') assert_frames_close(output, canon, rtol=rtol) From b12f9576626d4602838eded59cc34f859fc5a345 Mon Sep 17 00:00:00 2001 From: Maria Date: Tue, 27 Apr 2021 17:56:15 +0200 Subject: [PATCH 06/17] Update numeric range of subscripts --- pysd/py_backend/vensim/vensim2py.py | 46 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 65b5907b..c5b27771 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -271,9 +271,7 @@ def get_equation_components(equation_str, root_path=None): expression = ~r".*" # expression could be anything, at this point. keyword = ":" _ basic_id _ ":" - sequence_id = id_common+ numbers+ - id_common = ~r"[A-Z]"i - numbers = ~r"[0-9]" + sequence_id = _ basic_id _ subscript = basic_id / escape_group func = basic_id string = "\'" ( "\\\'" / ~r"[^\']"IU )* "\'" @@ -322,13 +320,13 @@ def visit_imported_subscript(self, n, vc): def visit_range(self, n, vc): subs_start = vc[2].strip() subs_end = vc[6].strip() - self.sequence, start, end = get_subscript_number_range(subs_start, subs_end) + self.sequence, start, end = get_subscript_numeric_range(subs_start, subs_end) for i in range(start, end+1): s = self.sequence + str(i) self.subscripts.append(s.strip()) def visit_value(self, n, vc): - self.subscripts.append(vc[1]) + self.subscripts.append(vc[1].strip()) def visit_name(self, n, vc): (name,) = vc @@ -367,10 +365,10 @@ def get_external_data(func_str, args_str, root_path): return f(*args, root=root_path).subscript -def get_subscript_number_range(subs_start, subs_end): +def get_subscript_numeric_range(subs_start, subs_end): """ With the first and the last subscript values of a subscript - numeric range, gets the common string of both and the + numeric range, gets the common prefix of both and the starting and ending number of the numeric range Parameters @@ -382,8 +380,8 @@ def get_subscript_number_range(subs_start, subs_end): Returns ------- - common: str - Common string of both subscripts + prefix: str + Common prefix of both subscripts num_start: int Sequence start number num_end: int @@ -396,21 +394,21 @@ def get_subscript_number_range(subs_start, subs_end): >>> get_subscript_number_range('sub15', 'sub30') ('sub', 15, 30) """ - subs_start_l = list(subs_start) - subs_end_l = list(subs_end) - common = "" - for i in range(len(subs_start)): - if(subs_start_l[i] == subs_end_l[i] and not(subs_start_l[i].isdigit())): - common = common + subs_start_l[i] - else: break - num_start = subs_start[i:] - num_end = subs_end[i:] - if(not(num_start.isdigit()) or not(num_end.isdigit()) ): raise ValueError("Format of subscript numeric range is not correct\n") - num_start = int(num_start) - num_end = int(num_end) - if(num_start>num_end): raise ValueError("The number of the first subscript value must be lower than the second subscript value in a subscript numeric range\n") - - return common, num_start, num_end + if(subs_start == subs_end): raise ValueError('Only different subscripts are valid in a numeric range') + + subs_start = re.findall('\d+|\D+', subs_start) + subs_end = re.findall('\d+|\D+', subs_end) + prefix_start = ''.join(subs_start[:-1]) + prefix_end = ''.join(subs_start[:-1]) + num_start = int(subs_start[-1]) + num_end = int(subs_end[-1]) + + if(not(prefix_start) or not(prefix_end)): raise ValueError('A numeric range must contain at least one letter') + if(num_start>num_end): raise ValueError('The number of the first subscript value must be lower than the second subscript value in a subscript numeric range\n') + if(prefix_start != prefix_end or subs_start[0].isdigit() or subs_end[0].isdigit()): raise ValueError('Only matching names ending in numbers are valid') + + return prefix_start, num_start, num_end + def parse_units(units_str): """ From 2c1ce8440cda7343e7539cd6fb7c3069bd70b680 Mon Sep 17 00:00:00 2001 From: Maria Date: Wed, 28 Apr 2021 19:08:38 +0200 Subject: [PATCH 07/17] A variable that is not an attribute was renamed --- pysd/py_backend/vensim/vensim2py.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index c5b27771..8f59f822 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -320,9 +320,9 @@ def visit_imported_subscript(self, n, vc): def visit_range(self, n, vc): subs_start = vc[2].strip() subs_end = vc[6].strip() - self.sequence, start, end = get_subscript_numeric_range(subs_start, subs_end) + prefix, start, end = get_subscript_numeric_range(subs_start, subs_end) for i in range(start, end+1): - s = self.sequence + str(i) + s = prefix + str(i) self.subscripts.append(s.strip()) def visit_value(self, n, vc): From 8b2a16ead394bfe3fd6054482b535675645bd4e5 Mon Sep 17 00:00:00 2001 From: Maria Date: Wed, 5 May 2021 17:57:25 +0200 Subject: [PATCH 08/17] Update test-models repository --- tests/test-models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-models b/tests/test-models index a132c9a3..d2a3c918 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit a132c9a3fcaf0950ba5fd97ed59f9a4aa9e46294 +Subproject commit d2a3c9183a351d5a081bcab80ed9b8dcda396c2e From b3afbd951c71b95c30e4ae1f586abe6e0f34d001 Mon Sep 17 00:00:00 2001 From: Maria Date: Wed, 5 May 2021 17:58:28 +0200 Subject: [PATCH 09/17] Updated prefix end of subscript numeric sequence --- pysd/py_backend/vensim/vensim2py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 8f59f822..0ba93aa2 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -399,7 +399,7 @@ def get_subscript_numeric_range(subs_start, subs_end): subs_start = re.findall('\d+|\D+', subs_start) subs_end = re.findall('\d+|\D+', subs_end) prefix_start = ''.join(subs_start[:-1]) - prefix_end = ''.join(subs_start[:-1]) + prefix_end = ''.join(subs_end[:-1]) num_start = int(subs_start[-1]) num_end = int(subs_end[-1]) From 601e2f757acbd9d609807707c23371512dc834e6 Mon Sep 17 00:00:00 2001 From: Maria Date: Thu, 6 May 2021 16:53:07 +0200 Subject: [PATCH 10/17] The conflict merged and tests bugs fixed --- pysd/py_backend/vensim/vensim2py.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 5c89c1f6..634b84db 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -276,7 +276,7 @@ def get_equation_components(equation_str, root_path=None): name = basic_id / escape_group literal_subscript = subscript _ ("," _ subscript _)* - imported_subscript = func _ "(" _ (string _ ","? _)* ")" + imported_subscript = imp_subs_func _ "(" _ (string _ ","? _)* ")" numeric_range = _ (range / value) _ ("," _ (range / value) _)* value = _ sequence_id _ range = "(" _ sequence_id _ "-" _ sequence_id _ ")" @@ -594,10 +594,7 @@ def parse_units(units_str): "original_name": "INVERT MATRIX"}, "get time value": {"name": "not_implemented_function", "module": "functions", - "original_name": "GET TIME VALUE"}, - "sample if true": {"name": "not_implemented_function", - "module": "functions", - "original_name": "SAMPLE IF TRUE"} + "original_name": "GET TIME VALUE"} } # list of fuctions that accept a dimension to apply over From 0e24ac723dbdb1c91bd33b540364ea81180402df Mon Sep 17 00:00:00 2001 From: Maria Date: Sat, 8 May 2021 12:17:43 +0200 Subject: [PATCH 11/17] Updated with new version of PySD --- pysd/py_backend/vensim/vensim2py.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 634b84db..777ab50a 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -379,18 +379,6 @@ def visit__(self, n, vc): 'kind': parse_object.kind, 'keyword': parse_object.keyword} - -def get_external_data(func_str, args_str, root_path): - """ - Gets the subscripts from external files calling the class external.ExtSubscript - """ - # The py model file must be recompiled if external file subscripts change. This could be avoided - # if we switch to function-defined subscript values instead of hard-coding them. - f = subscript_functions[func_str.lower()] - args = [x.strip().strip("\'") for x in args_str.split(',')] # todo: make this less fragile? - - return f(*args, root=root_path).subscript - def get_subscript_numeric_range(subs_start, subs_end): """ With the first and the last subscript values of a subscript From 5b68e6251fa7eaddbb69a678f42c57209280b555 Mon Sep 17 00:00:00 2001 From: Maria Date: Mon, 10 May 2021 16:34:02 +0200 Subject: [PATCH 12/17] Implement sample if true functionality as a Stateful object --- pysd/py_backend/builder.py | 111 ++++++++++++++++++++++ pysd/py_backend/functions.py | 142 +++++++++++----------------- pysd/py_backend/vensim/vensim2py.py | 26 ++--- 3 files changed, 176 insertions(+), 103 deletions(-) diff --git a/pysd/py_backend/builder.py b/pysd/py_backend/builder.py index 89f69daf..7c4e8fc9 100644 --- a/pysd/py_backend/builder.py +++ b/pysd/py_backend/builder.py @@ -565,6 +565,117 @@ def add_n_delay(identifier, delay_input, delay_time, initial_value, order, return "%s()" % stateful['py_name'], new_structure +def add_sample_if_true(identifier, condition, actual_value, initial_value, + subs, subscript_dict): + """ + Creates code to instantiate a stateful 'SampleIfTrue' object, + and provides reference to that object's output. + + Parameters + ---------- + identifier: basestring + the python-safe name of the stock + + condition: + Reference to another model element that is the condition to the + 'sample if true' function + + actual_value: + Can be a number (in string format) or a reference to another model + element which is calculated throughout the simulation at runtime. + + initial_value: + This is used to initialize the state of the sample if true function. + + subs: list of strings + List of strings of subscript indices that correspond to the + list of expressions, and collectively define the shape of the output + + subscript_dict: dictionary + Dictionary describing the possible dimensions of the stock's subscripts + + Returns + ------- + reference: basestring + reference to the sample if true object `__call__` method, + which will return the output of the sample if true process + + new_structure: list + list of element construction dictionaries for the builder to assemble + + """ + import_modules['functions'].add("SampleIfTrue") + + new_structure = [] + + if len(subs) == 0: + stateful_py_expr = 'SampleIfTrue(lambda: %s, lambda: %s,'\ + 'lambda: %s)' % (condition, actual_value, initial_value) + + else: + stateful_py_expr = 'SampleIfTrue(lambda: _condition_%s(),'\ + 'lambda: _input_%s(), lambda: _init_%s(),)' % ( + identifier, identifier, identifier) + # following elements not specified in the model file, but must exist + # create the sample if true initialization element + new_structure.append({ + 'py_name': '_init_%s' % identifier, + 'real_name': 'Implicit', + 'kind': 'setup', # not specified in the model file, but must exist + 'py_expr': initial_value, + 'subs': subs, + 'doc': 'Provides initial value for %s function' % identifier, + 'unit': 'See docs for %s' % identifier, + 'lims': 'None', + 'eqn': 'None', + 'arguments': '' + }) + + new_structure.append({ + 'py_name': '_condition_%s' % identifier, + 'real_name': 'Implicit', + 'kind': 'component', + 'doc': 'Provides condition for %s function' % identifier, + 'subs': subs, + 'unit': 'See docs for %s' % identifier, + 'lims': 'None', + 'eqn': 'None', + 'py_expr': condition, + 'arguments': '' + }) + + new_structure.append({ + 'py_name': '_input_%s' % identifier, + 'real_name': 'Implicit', + 'kind': 'component', + 'doc': 'Provides input for %s function' % identifier, + 'subs': subs, + 'unit': 'See docs for %s' % identifier, + 'lims': 'None', + 'eqn': 'None', + 'py_expr': actual_value, + 'arguments': '' + }) + + # describe the stateful object + stateful = { + 'py_name': '_sample_if_true_%s' % identifier, + 'real_name': 'Sample if true of %s' % identifier, + 'doc': 'Initial value: %s \n Input: %s \n Condition: %s' % ( + initial_value, actual_value, condition), + 'py_expr': stateful_py_expr, + 'unit': 'None', + 'lims': 'None', + 'eqn': 'None', + 'subs': '', + 'kind': 'stateful', + 'arguments': '' + } + new_structure.append(stateful) + + return "%s()" % stateful['py_name'], new_structure + + def add_n_smooth(identifier, smooth_input, smooth_time, initial_value, order, subs, subscript_dict): """ diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index 3e272814..9f881e9f 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -170,6 +170,33 @@ def ddt(self): inflows[0] = self.input_func() return inflows - outflows +class SampleIfTrue(Stateful): + def __init__(self, condition, actual_value, initial_value): + """ + + Parameters + ---------- + condition: function + actual_value: function + initial_value: function + """ + super().__init__() + self.condition = condition + self.actual_value = actual_value + self.initial_value = initial_value + + def initialize(self): + self.state = self.initial_value() + if isinstance(self.state, xr.DataArray): + self.shape_info = {'dims': self.state.dims, + 'coords': self.state.coords} + + def __call__(self): + self.state = sample_if_true(self.condition(), self.actual_value(), self.state) + return self.state + + def ddt(self): + return 0 class Smooth(Stateful): def __init__(self, smooth_input, smooth_time, initial_value, order): @@ -1091,6 +1118,34 @@ def if_then_else(condition, val_if_true, val_if_false): return val_if_true() if condition else val_if_false() +def sample_if_true(condition, actual_value, saved_value): + """ + Implements Vensim's SAMPLE IF TRUE function. + + Parameters + ---------- + condition: bool or xarray.DataArray of bools + actual_value: int, float or xarray.DataArray + Value to return when condition is true. + saved_value: int, float or xarray.DataArray + Value to return when condition is false. + + Returns + ------- + The value depending on the condition. + + """ + if isinstance(condition, xr.DataArray): + if condition.all(): + return actual_value + elif not condition.any(): + return saved_value + + return xr.where(condition, actual_value, saved_value) + + return actual_value if condition else saved_value + + def xidz(numerator, denominator, value_if_denom_is_zero): """ Implements Vensim's XIDZ function. @@ -1202,93 +1257,6 @@ def random_uniform(m, x, s): return np.random.uniform(m, x) -# dictionary to store the values from SAMPLE IF TRUE function -saved_value = {} - -def make_da(rows, cols, initial_value): - """ - Returns a DataArray with the coordinates - of the rows and cols. - DataArray values are initialized with the initial_value. - It is used in SAMPLE IF TRUE function, to create the proper dimension saved value. - - Parameters - ---------- - rows: float or xarray.DataArray - Represents the row dimension of the new DataArray - cols: xarray.DataArray - Represents the col dimension of the new DataArray - initial_value: float or xarray.DataArray - Include the values to initialize the new DataArray - - Returns - ------- - A new DataArray with proper rows and cols coordinates, - initialized with initial_value - - """ - if(isinstance(initial_value,xr.DataArray)): - array = np.array([[initial_value.data[e] for i in range(0, len(rows.values))] for e in range(0,len(cols.values))]) - elif(isinstance(rows, xr.DataArray)): - array = np.array([[initial_value for i in range(0, len(rows.values))] for i in range(0,len(cols.values))]) - else: - array = np.array([initial_value for i in range(0,len(cols.values))]) - - coords = {dim: cols.coords[dim] for dim in cols.dims} - dims = cols.dims - if(isinstance(rows, xr.DataArray)): - coords.update({dim: rows.coords[dim] for dim in rows.dims}) - dims += rows.dims - return xr.DataArray(data=array, coords=coords, dims=dims) - -def sample_if_true(time, condition, actual_value, initial_value, var_name): - """ - Implements Vensim's SAMPLE IF TRUE function. - - Parameters - ---------- - condition: bool or xarray.DataArray - actual_value: float or xarray.DataArray - Value to return when condition is true. - initial_value: float or xarray.DataArray - Value to return when condition is false. - var_name: str - Represents the SAMPLE IF TRUE function in the whole model. - - Returns - ------- - float or xarray.DataArray - Actual_value when condition is true and saved this value - in saved_value dictionary. - Returns the last saved value when condicion is false. - Saved value is initialized with initial_value in the first step of simulation. - """ - global saved_value - t = time() - - if(t==0): - if(not(isinstance(condition,xr.DataArray))): - saved_value[var_name] = initial_value - else: - saved_value[var_name] = make_da(actual_value, condition, initial_value) - - if isinstance(condition, xr.DataArray): - if condition.all(): - for i in range(0,len(saved_value[var_name].values)): - saved_value[var_name].values[i]=actual_value - return saved_value[var_name] - elif not condition.any(): - return saved_value[var_name] - - for i in range(0, len(condition)): - if condition.values[i]: - saved_value[var_name][i]=actual_value - return xr.where(condition, actual_value, saved_value[var_name]) - - if condition: - saved_value[var_name] = actual_value - - return saved_value[var_name] def incomplete(*args): warnings.warn( diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 777ab50a..793c0611 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -505,17 +505,6 @@ def parse_units(units_str): ], "module": "functions" }, - "sample if true": { - "name": "sample_if_true", - "parameters": [ - {"name": 'time', "type": 'time'}, - {"name": 'condition'}, - {"name": 'val_if_true'}, - {"name": 'val_if_false'}, - {"name": 'var_name'} - ], - "module": "functions" - }, "step": { "name": "step", "parameters": [ @@ -691,7 +680,16 @@ def parse_units(units_str): subs=element['subs'], subscript_dict=subscript_dict ), - + + "sample if true": lambda element, subscript_dict, args: builder.add_sample_if_true( + identifier=element['py_name'], + condition=args[0], + actual_value=args[1], + initial_value=args[2], + subs=element['subs'], + subscript_dict=subscript_dict + ), + "smooth": lambda element, subscript_dict, args: builder.add_n_smooth( identifier=element['py_name'], smooth_input=args[0], @@ -970,10 +968,6 @@ def visit_call(self, n, vc): if self.apply_dim and function_name in vectorial_funcs: arguments += ["dim="+str(tuple(self.apply_dim))] self.apply_dim = set() - - # add name of variable to sample if true function arguments - if(isinstance(functions[function_name],dict) and functions[function_name]['name'] == "sample_if_true"): - arguments.append(element['py_name']) return builder.build_function_call(functions[function_name], arguments) From c095ee1564834d7746c4fc1f815e16a3d2a9cd32 Mon Sep 17 00:00:00 2001 From: Maria Date: Mon, 10 May 2021 17:11:45 +0200 Subject: [PATCH 13/17] Add more information in the errors of visit_numeric_range and removed the function get_subscript_numeric_range and the code has been added to the visitor --- pysd/py_backend/vensim/vensim2py.py | 63 ++++++++--------------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/pysd/py_backend/vensim/vensim2py.py b/pysd/py_backend/vensim/vensim2py.py index 793c0611..22d9874a 100644 --- a/pysd/py_backend/vensim/vensim2py.py +++ b/pysd/py_backend/vensim/vensim2py.py @@ -335,9 +335,23 @@ def visit_imported_subscript(self, n, vc): def visit_range(self, n, vc): subs_start = vc[2].strip() subs_end = vc[6].strip() - prefix, start, end = get_subscript_numeric_range(subs_start, subs_end) - for i in range(start, end+1): - s = prefix + str(i) + if(subs_start == subs_end): raise ValueError('Only different subscripts are valid in a numeric range, error in expression:\n\t %s\n' % (equation_str)) + + # get the common prefix and the starting and + # ending number of the numeric range + subs_start = re.findall('\d+|\D+', subs_start) + subs_end = re.findall('\d+|\D+', subs_end) + prefix_start = ''.join(subs_start[:-1]) + prefix_end = ''.join(subs_end[:-1]) + num_start = int(subs_start[-1]) + num_end = int(subs_end[-1]) + + if(not(prefix_start) or not(prefix_end)): raise ValueError('A numeric range must contain at least one letter, error in expression:\n\t %s\n' % (equation_str)) + if(num_start>num_end): raise ValueError('The number of the first subscript value must be lower than the second subscript value in a subscript numeric range, error in expression:\n\t %s\n'% (equation_str)) + if(prefix_start != prefix_end or subs_start[0].isdigit() or subs_end[0].isdigit()): raise ValueError('Only matching names ending in numbers are valid, error in expression:\n\t %s\n'% (equation_str)) + + for i in range(num_start, num_end+1): + s = prefix_start + str(i) self.subscripts.append(s.strip()) def visit_value(self, n, vc): @@ -379,49 +393,6 @@ def visit__(self, n, vc): 'kind': parse_object.kind, 'keyword': parse_object.keyword} -def get_subscript_numeric_range(subs_start, subs_end): - """ - With the first and the last subscript values of a subscript - numeric range, gets the common prefix of both and the - starting and ending number of the numeric range - - Parameters - ---------- - subs_start: str - Represents the first subscript value in the numeric range - subs_end: str - Represents the last subscript value in the numeric range - - Returns - ------- - prefix: str - Common prefix of both subscripts - num_start: int - Sequence start number - num_end: int - Sequence end number - - Examples - -------- - >>> get_subscript_number_range('Layer1', 'Layer5') - ('Layer', 1, 5) - >>> get_subscript_number_range('sub15', 'sub30') - ('sub', 15, 30) - """ - if(subs_start == subs_end): raise ValueError('Only different subscripts are valid in a numeric range') - - subs_start = re.findall('\d+|\D+', subs_start) - subs_end = re.findall('\d+|\D+', subs_end) - prefix_start = ''.join(subs_start[:-1]) - prefix_end = ''.join(subs_end[:-1]) - num_start = int(subs_start[-1]) - num_end = int(subs_end[-1]) - - if(not(prefix_start) or not(prefix_end)): raise ValueError('A numeric range must contain at least one letter') - if(num_start>num_end): raise ValueError('The number of the first subscript value must be lower than the second subscript value in a subscript numeric range\n') - if(prefix_start != prefix_end or subs_start[0].isdigit() or subs_end[0].isdigit()): raise ValueError('Only matching names ending in numbers are valid') - - return prefix_start, num_start, num_end def parse_units(units_str): """ From ee9db93adcf35bb91f8dd076987a185d964043d8 Mon Sep 17 00:00:00 2001 From: Maria Date: Mon, 10 May 2021 18:10:19 +0200 Subject: [PATCH 14/17] Updated version of pysd --- pysd/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysd/_version.py b/pysd/_version.py index c1afe526..17949574 100644 --- a/pysd/_version.py +++ b/pysd/_version.py @@ -1,2 +1,2 @@ -__version__ = "1.2.0" +__version__ = "1.3.0" From 35243c856194c626a0da59fe066f69fda654685c Mon Sep 17 00:00:00 2001 From: Maria Date: Mon, 10 May 2021 18:16:05 +0200 Subject: [PATCH 15/17] Updated sample if true class, deleted sample_if_true function --- pysd/py_backend/functions.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index 9f881e9f..3736f646 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -192,7 +192,7 @@ def initialize(self): 'coords': self.state.coords} def __call__(self): - self.state = sample_if_true(self.condition(), self.actual_value(), self.state) + self.state = if_then_else(self.condition(), self.actual_value, lambda: self.state) return self.state def ddt(self): @@ -1118,34 +1118,6 @@ def if_then_else(condition, val_if_true, val_if_false): return val_if_true() if condition else val_if_false() -def sample_if_true(condition, actual_value, saved_value): - """ - Implements Vensim's SAMPLE IF TRUE function. - - Parameters - ---------- - condition: bool or xarray.DataArray of bools - actual_value: int, float or xarray.DataArray - Value to return when condition is true. - saved_value: int, float or xarray.DataArray - Value to return when condition is false. - - Returns - ------- - The value depending on the condition. - - """ - if isinstance(condition, xr.DataArray): - if condition.all(): - return actual_value - elif not condition.any(): - return saved_value - - return xr.where(condition, actual_value, saved_value) - - return actual_value if condition else saved_value - - def xidz(numerator, denominator, value_if_denom_is_zero): """ Implements Vensim's XIDZ function. From 494e57198a9ec84e1dc94b43177dd4d1d4b41543 Mon Sep 17 00:00:00 2001 From: Maria Date: Mon, 10 May 2021 18:23:02 +0200 Subject: [PATCH 16/17] SampleIfTrue added --- docs/development/supported_vensim_functions.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/development/supported_vensim_functions.rst b/docs/development/supported_vensim_functions.rst index 37a14fb3..6ec979b6 100644 --- a/docs/development/supported_vensim_functions.rst +++ b/docs/development/supported_vensim_functions.rst @@ -69,6 +69,8 @@ +------------------------------+------------------------------+ | DELAY N | functions.Delay | +------------------------------+------------------------------+ +| SAMPLE IF TRUE | functions.SampleIfTrue | ++------------------------------+------------------------------+ | SMOOTH3I | functions.Smooth | +------------------------------+------------------------------+ | SMOOTH3 | functions.Smooth | From 10338cae89e5958aedb9c5e61610997599b86c0f Mon Sep 17 00:00:00 2001 From: Eneko Martin Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Tue, 11 May 2021 09:45:13 +0200 Subject: [PATCH 17/17] Add update method to SampleIfTrue Add update method to SampleIfTrue with a pass, so when trying to update SampleIfTrue nothing will be done. Improves reliability and speed. --- pysd/py_backend/functions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index 3736f646..14bafbeb 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -198,6 +198,10 @@ def __call__(self): def ddt(self): return 0 + def update(self, state): + # this doesn't change once it's set up. + pass + class Smooth(Stateful): def __init__(self, smooth_input, smooth_time, initial_value, order): super().__init__()