Better and/or operators and fixes on tests

pysd/py_backend/builder.py

@@ -670,7 +670,7 @@ def add_incomplete(var_name, dependencies):
                   SyntaxWarning, stacklevel=2)
 
     # first arg is `self` reference
-    return "functions.incomplete(%s)" % ', '.join(dependencies[1:]), []
+    return "functions.incomplete(%s)" % ', '.join(dependencies), []
 
 
 def build_function_call(function_def, user_arguments):

pysd/py_backend/vensim/vensim2py.py

@@ -208,7 +208,7 @@ def _include_common_grammar(source_grammar):
     """.format(source_grammar=source_grammar, common_grammar=common_grammar)
 
 
-def get_equation_components(equation_str, root_path):
+def get_equation_components(equation_str, root_path=None):
     """
     Breaks down a string representing only the equation part of a model element.
     Recognizes the various types of model elements that may exist, and identifies them.
@@ -701,7 +701,7 @@ def parse_general_expression(element, namespace=None, subscript_dict=None, macro
     in_ops = {
         "+": "+", "-": "-", "*": "*", "/": "/", "^": "**", "=": "==", "<=": "<=", "<>": "!=",
         "<": "<", ">=": ">=", ">": ">",
-        ":and:": " and ", ":or:": " or "}  # spaces important for word-based operators
+        ":and:": " & ", ":or:": " | "}  # spaces perhaps important?
 
     pre_ops = {
         "-": "-", ":not:": " not ",  # spaces important for word-based operators
@@ -722,9 +722,8 @@ def parse_general_expression(element, namespace=None, subscript_dict=None, macro
 
     expression_grammar = r"""
     expr_type = array / expr / empty
-    expr = _ pre_oper? _ (lookup_def / build_call / macro_call / call / lookup_call / parens / number / string / reference) _ in_oper_expr?
+    expr = _ pre_oper? _ (lookup_def / build_call / macro_call / call / lookup_call / parens / number / string / reference) _ (in_oper _ expr)?
 
-    in_oper_expr = (in_oper _ expr)
     lookup_def = ~r"(WITH\ LOOKUP)"I _ "(" _ expr _ "," _ "(" _  ("[" ~r"[^\]]*" "]" _ ",")?  ( "(" _ expr _ "," _ expr _ ")" _ ","? _ )+ _ ")" _ ")"
     lookup_call = (id _ subscript_list?) _ "(" _ (expr _ ","? _)* ")"  # these don't need their args parsed...
     call = func _ "(" _ (expr _ ","? _)* ")"  # these don't need their args parsed...
@@ -785,30 +784,10 @@ def visit_expr_type(self, n, vc):
             self.translation = s
 
         def visit_expr(self, n, vc):
-            if self.in_oper:
-                # This is rather inelegant, and possibly could be better implemented with a serious reorganization
-                # of the grammar specification for general expressions.
-                args = [x for x in vc if len(x.strip())]
-                if len(args) == 3:
-                    args = [''.join(args[0:2]), args[2]]
-                if self.in_oper  == ' and ':
-                    s = 'functions.and_(%s)' % ','.join(args)
-                elif self.in_oper == ' or ':
-                    s = 'functions.or_(%s)' % ','.join(args)
-                else:
-                    s = self.in_oper.join(args)
-                self.in_oper = None
-            else:
-                s = ''.join(filter(None, vc)).strip()
+            s = ''.join(filter(None, vc)).strip()
             self.translation = s
             return s
 
-        def visit_in_oper_expr(self, n, vc):
-            # We have to pull out the internal operator because the Python "and" and "or" operator do not work with
-            # numpy arrays or xarray DataArrays. We will later replace it with the functions.and_ or functions.or_.
-            self.in_oper = vc[0]
-            return ''.join(filter(None, vc[1:])).strip()
-
         def visit_call(self, n, vc):
             self.kind = 'component'
             function_name = vc[0].lower()

tests/integration_test_vensim_pathway.py

@@ -137,7 +137,8 @@ def test_lookups(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/lookups/test_lookups.mdl')
         assert_frames_close(output, canon, rtol=rtol)
-
+
+    @unittest.skip('File not found')
     def test_lookups_without_range(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/lookups_without_range/test_lookups_without_range.mdl')
@@ -250,6 +251,7 @@ def test_sqrt(self):
         output, canon = runner('test-models/tests/sqrt/test_sqrt.mdl')
         assert_frames_close(output, canon, rtol=rtol)
 
+    @unittest.skip('File not found')
     def test_subscript_multiples(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/subscript_multiples/test_multiple_subscripts.mdl')
@@ -323,25 +325,25 @@ def test_subscript_mixed_assembly(self):
         output, canon = runner('test-models/tests/subscript_mixed_assembly/test_subscript_mixed_assembly.mdl')
         assert_frames_close(output, canon, rtol=rtol)
 
-    @unittest.skip('in branch')
+    # @unittest.skip('in branch')
     def test_subscript_selection(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/subscript_selection/subscript_selection.mdl')
         assert_frames_close(output, canon, rtol=rtol)
 
-    @unittest.skip('failing in py3')
+    # @unittest.skip('failing in py3')
     def test_subscript_subranges(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/subscript_subranges/test_subscript_subrange.mdl')
         assert_frames_close(output, canon, rtol=rtol)
 
-    @unittest.skip('failing in py3')
+    # @unittest.skip('failing in py3')
     def test_subscript_subranges_equal(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/subscript_subranges_equal/test_subscript_subrange_equal.mdl')
         assert_frames_close(output, canon, rtol=rtol)
 
-    @unittest.skip('in branch')
+    # @unittest.skip('in branch')
     def test_subscript_switching(self):
         from.test_utils import runner, assert_frames_close
         output, canon = runner('test-models/tests/subscript_switching/subscript_switching.mdl')

tests/unit_test_vensim2py.py

@@ -71,7 +71,7 @@ def test_basics(self):
         from pysd.py_backend.vensim.vensim2py import get_equation_components
         self.assertEqual(
             get_equation_components(r'constant = 25'),
-            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': 'constant'}
+            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': 'constant', 'keyword': None}
         )
 
     def test_equals_handling(self):
@@ -80,7 +80,7 @@ def test_equals_handling(self):
         self.assertEqual(
             get_equation_components(r'Boolean = IF THEN ELSE(1 = 1, 1, 0)'),
             {'expr': 'IF THEN ELSE(1 = 1, 1, 0)', 'kind': 'component', 'subs': [],
-             'real_name': 'Boolean'}
+             'real_name': 'Boolean', 'keyword': None}
         )
 
     def test_whitespace_handling(self):
@@ -89,76 +89,76 @@ def test_whitespace_handling(self):
         self.assertEqual(
             get_equation_components(r'''constant\t =
                                                         \t25\t '''),
-            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': 'constant'}
+            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': 'constant', 'keyword': None}
         )
 
         # test eliminating vensim's line continuation character
         self.assertEqual(
             get_equation_components(r"""constant [Sub1, \\
                                      Sub2] = 10, 12; 14, 16;"""),
             {'expr': '10, 12; 14, 16;', 'kind': 'component', 'subs': ['Sub1', 'Sub2'],
-             'real_name': 'constant'}
+             'real_name': 'constant', 'keyword': None}
         )
 
     def test_subscript_definition_parsing(self):
         from pysd.py_backend.vensim.vensim2py import get_equation_components
         self.assertEqual(
             get_equation_components(r'''Sub1: Entry 1, Entry 2, Entry 3 '''),
             {'expr': None, 'kind': 'subdef', 'subs': ['Entry 1', 'Entry 2', 'Entry 3'],
-             'real_name': 'Sub1'}
+             'real_name': 'Sub1', 'keyword': None}
         )
 
     def test_subscript_references(self):
         from pysd.py_backend.vensim.vensim2py import get_equation_components
         self.assertEqual(
             get_equation_components(r'constant [Sub1, Sub2] = 10, 12; 14, 16;'),
             {'expr': '10, 12; 14, 16;', 'kind': 'component', 'subs': ['Sub1', 'Sub2'],
-             'real_name': 'constant'}
+             'real_name': 'constant', 'keyword': None}
         )
 
         self.assertEqual(
             get_equation_components(r'function [Sub1] = other function[Sub1]'),
             {'expr': 'other function[Sub1]', 'kind': 'component', 'subs': ['Sub1'],
-             'real_name': 'function'}
+             'real_name': 'function', 'keyword': None}
         )
 
         self.assertEqual(
             get_equation_components(r'constant ["S1,b", "S1,c"] = 1, 2; 3, 4;'),
             {'expr': '1, 2; 3, 4;', 'kind': 'component', 'subs': ['"S1,b"', '"S1,c"'],
-             'real_name': 'constant'}
+             'real_name': 'constant', 'keyword': None}
         )
 
         self.assertEqual(
             get_equation_components(r'constant ["S1=b", "S1=c"] = 1, 2; 3, 4;'),
             {'expr': '1, 2; 3, 4;', 'kind': 'component', 'subs': ['"S1=b"', '"S1=c"'],
-             'real_name': 'constant'}
+             'real_name': 'constant', 'keyword': None}
         )
 
     def test_lookup_definitions(self):
         from pysd.py_backend.vensim.vensim2py import get_equation_components
         self.assertEqual(
             get_equation_components(r'table([(0,-1)-(45,1)],(0,0),(5,0))'),
             {'expr': '([(0,-1)-(45,1)],(0,0),(5,0))', 'kind': 'lookup', 'subs': [],
-             'real_name': 'table'}
+             'real_name': 'table', 'keyword': None}
         )
 
         self.assertEqual(
             get_equation_components(r'table2 ([(0,-1)-(45,1)],(0,0),(5,0))'),
             {'expr': '([(0,-1)-(45,1)],(0,0),(5,0))', 'kind': 'lookup', 'subs': [],
-             'real_name': 'table2'}
+             'real_name': 'table2', 'keyword': None}
         )
 
     def test_pathological_names(self):
         from pysd.py_backend.vensim.vensim2py import get_equation_components
         self.assertEqual(
             get_equation_components(r'"silly-string" = 25'),
-            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': '"silly-string"'}
+            {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': '"silly-string"', 'keyword': None}
         )
 
         self.assertEqual(
             get_equation_components(r'"pathological\\-string" = 25'),
             {'expr': '25', 'kind': 'component', 'subs': [],
-             'real_name': r'"pathological\\-string"'}
+             'real_name': r'"pathological\\-string"', 'keyword': None}
         )
 
 
@@ -328,7 +328,7 @@ def test_subscript_3d_depth(self):
         self.assertEqual(a.loc[{'Dim1': 'A', 'Dim2': 'D'}], 1)
         self.assertEqual(a.loc[{'Dim1': 'B', 'Dim2': 'E'}], 4)
 
-    @unittest.skip('in branch')
+    # unittest.skip('in branch')
     def test_subscript_reference(self):
         from pysd.py_backend.vensim.vensim2py import parse_general_expression
         res = parse_general_expression({'expr': 'Var A[Dim1, Dim2]'},
@@ -342,16 +342,16 @@ def test_subscript_reference(self):
                                      {'Var B': 'var_b'},
                                      {'Dim1': ['A', 'B'],
                                       'Dim2': ['C', 'D', 'E']})
-        self.assertEqual(res[0]['py_expr'], "var_b().loc[{'Dim2': ['C']}]")
+        self.assertEqual(res[0]['py_expr'], "var_b().loc[{'Dim2': ['C']}].squeeze()")
 
         res = parse_general_expression({'expr': 'Var C[Dim1, C, H]'},
                                      {'Var C': 'var_c'},
                                      {'Dim1': ['A', 'B'],
                                       'Dim2': ['C', 'D', 'E'],
                                       'Dim3': ['F', 'G', 'H', 'I']})
-        self.assertEqual(res[0]['py_expr'], "var_c().loc[{'Dim2': ['C'], 'Dim3': ['H']}]")
+        self.assertEqual(res[0]['py_expr'], "var_c().loc[{'Dim2': ['C'], 'Dim3': ['H']}].squeeze()")
 
-    @unittest.skip('in branch')
+    # @unittest.skip('in branch')
     def test_subscript_ranges(self):
         from pysd.py_backend.vensim.vensim2py import parse_general_expression
         res = parse_general_expression({'expr': 'Var D[Range1]'},
@@ -360,10 +360,10 @@ def test_subscript_ranges(self):
                                       'Range1': ['C', 'D', 'E']})
         self.assertEqual(res[0]['py_expr'], "var_c().loc[{'Dim1': ['C', 'D', 'E']}]")
 
-    @unittest.skip('need to write this properly')
+    # @unittest.skip('need to write this properly')
     def test_incomplete_expression(self):
         from pysd.py_backend.vensim.vensim2py import parse_general_expression
-        res = parse_general_expression({'expr': 'A FUNCTION OF(Unspecified Eqn,Var A,Var B)'},
+        res = parse_general_expression({'expr': 'A FUNCTION OF(Unspecified Eqn,Var A,Var B)', 'real_name': 'something'},
                                        {'Unspecified Eqn': 'unspecified_eqn',
                                         'Var A': 'var_a',
                                         'Var B': 'var_b'})