Native types instead of strings (#617)

In Snowfakery 2, all formulas are evaluated to either a string or
number. In Snowfakery 3, other types are possible. Dates are the
most prominent reason to prefer this.

examples/native_types_differences.yml

@@ -0,0 +1,6 @@
+# snowfakery examples/native_types_differences.yml --plugin-option snowfakery_version 3
+- snowfakery_version: 3
+- object: Example
+  fields:
+    y: ${{datetime(year=2000, month=1, day=1)}}
+    z: ${{y.date()}}

schema/snowfakery_recipe.jsonschema.json

@@ -23,6 +23,9 @@
       },
       {
         "$ref": "#/$defs/option"
+      },
+      {
+        "$ref": "#/$defs/version"
       }
     ]
   },
@@ -226,10 +229,20 @@
             ]
           }
         }
+      }
+    },
+    "version": {
+      "title": "Version Declaration",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "snowfakery_version": {
+          "description": "The Snowfakery version",
+          "type": "integer",
+          "enum": [2, 3]
+        }
       },
-      "required": [
-        "macro"
-      ]
+      "required": ["snowfakery_version"]
     }
   }
 }

snowfakery/data_generator.py

@@ -7,6 +7,8 @@
 from faker.providers import BaseProvider as FakerProvider
 from click.utils import LazyFile
 
+from snowfakery.standard_plugins.SnowfakeryVersion import SnowfakeryVersion
+
 from .data_gen_exceptions import DataGenNameError
 from .output_streams import DebugOutputStream, OutputStream
 from .parse_recipe_yaml import parse_recipe
@@ -143,8 +145,12 @@ def generate(
     faker_providers, snowfakery_plugins = process_plugins(parse_result.plugins)
 
     snowfakery_plugins.setdefault("UniqueId", UniqueId)
+    snowfakery_plugins.setdefault("SnowfakeryVersion", SnowfakeryVersion)
+    plugin_options = plugin_options or {}
+    if parse_result.version:
+        plugin_options["snowfakery_version"] = parse_result.version
 
-    plugin_options = process_plugins_options(snowfakery_plugins, plugin_options or {})
+    plugin_options = process_plugins_options(snowfakery_plugins, plugin_options)
 
     # figure out how it relates to CLI-supplied generation variables
     options, extra_options = merge_options(
@@ -203,7 +209,6 @@ def process_plugins_options(
     e.g. the option name that the user specifies on the CLI or API is just "org_name"
          but we use the long name internally to avoid clashing with the
          user's variable names."""
-
     allowed_options = collect_allowed_plugin_options(tuple(plugins.values()))
     plugin_options = {}
     for option in allowed_options:

snowfakery/data_generator_runtime.py

@@ -7,6 +7,7 @@
 from warnings import warn
 
 import jinja2
+from jinja2 import nativetypes
 import yaml
 
 from .utils.template_utils import FakerTemplateLibrary
@@ -234,7 +235,21 @@ def deserialize_dict_of_object_rows(dct):
 
 
 class JinjaTemplateEvaluatorFactory:
-    def __init__(self):
+    def __init__(self, native_types: bool):
+        if native_types:
+            self.compilers = [
+                nativetypes.NativeEnvironment(
+                    block_start_string="${%",
+                    block_end_string="%}",
+                    variable_start_string="${{",
+                    variable_end_string="}}",
+                )
+            ]
+
+            return
+
+        # TODO: Delete this old code_path when the
+        #       transition to native_types is complete.
         self.compilers = [
             jinja2.Environment(
                 block_start_string="${%",
@@ -325,6 +340,12 @@ def __init__(
         self.parent_application = parent_application
         self.instance_states = {}
         self.filter_row_values = self.filter_row_values_normal
+        snowfakery_version = self.options.get(
+            "snowfakery.standard_plugins.SnowfakeryVersion.snowfakery_version", 2
+        )
+        assert snowfakery_version in (2, 3)
+        native_types = snowfakery_version == 3
+        self.template_evaluator_factory = JinjaTemplateEvaluatorFactory(native_types)
 
     def execute(self):
         self.current_context = RuntimeContext(interpreter=self)
@@ -432,7 +453,6 @@ class RuntimeContext:
     but internally its mostly just proxying to other classes."""
 
     obj: Optional[ObjectRow] = None
-    template_evaluator_recipe = JinjaTemplateEvaluatorFactory()
     current_template = None
     local_vars = None
     unique_context_identifier = None
@@ -525,7 +545,7 @@ def output_stream(self):
         return self.interpreter.output_stream
 
     def get_evaluator(self, definition: str):
-        return self.template_evaluator_recipe.get_evaluator(definition)
+        return self.interpreter.template_evaluator_factory.get_evaluator(definition)
 
     @property
     def evaluation_namespace(self):

snowfakery/parse_recipe_yaml.py

@@ -36,13 +36,19 @@
 
 class ParseResult:
     def __init__(
-        self, options, tables: Mapping, statements: Sequence, plugins: Sequence = ()
+        self,
+        options,
+        tables: Mapping,
+        statements: Sequence,
+        plugins: Sequence = (),
+        version: int = None,
     ):
         self.options = options
         self.tables = tables
         self.statements = statements
         self.templates = [obj for obj in statements if isinstance(obj, ObjectTemplate)]
         self.plugins = plugins
+        self.version = version
 
 
 class TableInfo:
@@ -604,12 +610,14 @@ def parse_included_files(path: Path, data: List, context: ParseContext):
     "plugin": "plugin",
     "object": "statement",
     "var": "statement",
+    "snowfakery_version": "snowfakery_version",
 }
 
 
 def categorize_top_level_objects(data: List, context: ParseContext):
     """Look at all of the top-level declarations and categorize them"""
     top_level_collections: Dict = {
+        "snowfakery_version": [],
         "option": [],
         "include_file": [],
         "macro": [],
@@ -655,13 +663,37 @@ def parse_top_level_elements(path: Path, data: List, context: ParseContext):
     context.plugins.extend(
         resolve_plugins(plugin_specs, search_paths=[plugin_near_recipe])
     )
+    context.version = parse_version(top_level_objects["snowfakery_version"], context)
     statements.extend(top_level_objects["statement"])
     for pluginbase, plugin in context.plugins:
         if pluginbase == ParserMacroPlugin:
             context.parser_macros_plugins[plugin.__name__] = plugin()
     return statements
 
 
+def parse_version(version_declarations: T.List[T.Dict], context: ParseContext):
+    if version_declarations:
+        base_version = version_declarations[0]["snowfakery_version"]
+        mismatched_versions = [
+            obj
+            for obj in version_declarations
+            if obj["snowfakery_version"] != base_version
+        ]
+        if mismatched_versions:
+            with context.change_current_parent_object(version_declarations[1]):
+                raise exc.DataGenSyntaxError(
+                    "Cannot have multiple conflicting versions in the same recipe: ",
+                    **context.line_num(),
+                )
+        if base_version not in (2, 3):
+            with context.change_current_parent_object(version_declarations[0]):
+                raise exc.DataGenSyntaxError(
+                    "Version must be 2 or 3: ",
+                    **context.line_num(),
+                )
+        return base_version
+
+
 def parse_file(stream: IO[str], context: ParseContext) -> List[Dict]:
     stream_name = getattr(stream, "name", None)
     if stream_name:
@@ -754,4 +786,10 @@ def parse_recipe(
             statements, tables, update_input_file, update_passthrough_fields
         )
 
-    return ParseResult(context.options, tables, statements, plugins=context.plugins)
+    return ParseResult(
+        context.options,
+        tables,
+        statements,
+        plugins=context.plugins,
+        version=context.version,
+    )

snowfakery/standard_plugins/SnowfakeryVersion.py

@@ -0,0 +1,12 @@
+from snowfakery import SnowfakeryPlugin
+from snowfakery.plugins import PluginOption
+
+plugin_options_version = (
+    "snowfakery.standard_plugins.SnowfakeryVersion.snowfakery_version"
+)
+
+
+class SnowfakeryVersion(SnowfakeryPlugin):
+    allowed_options = [
+        PluginOption(plugin_options_version, int),
+    ]

tests/errors/bad_version.recipe.yml

@@ -0,0 +1,2 @@
+- snowfakery_version: 4
+- object: Example

tests/test_dates.py

@@ -0,0 +1,40 @@
+import pytest
+from io import StringIO
+from snowfakery import generate_data
+from snowfakery import data_gen_exceptions as exc
+
+
+class TestDates:
+    def test_old_dates_as_strings(self, generated_rows):
+        yaml = """
+        - object: OBJ
+          fields:
+            basedate: ${{datetime(year=2000, month=1, day=1)}}
+            dateplus: ${{basedate + "XYZZY"}}
+        """
+        generate_data(StringIO(yaml))
+        date = generated_rows.table_values("OBJ", 1, "dateplus")
+        assert date.startswith("2000")
+        assert date.endswith("XYZZY")
+
+    def test_date_math__native_types(self, generated_rows):
+        yaml = """
+        - object: OBJ
+          fields:
+            basedate: ${{datetime(year=2000, month=1, day=1)}}
+            dateplus: ${{basedate + relativedelta(years=22)}}
+        """
+        generate_data(StringIO(yaml), plugin_options={"snowfakery_version": 3})
+        date = generated_rows.table_values("OBJ", 1, "dateplus")
+        assert date == "2022-01-01T00:00:00+00:00"
+
+    def test_date_math__native_types__error(self, generated_rows):
+        yaml = """
+        - object: OBJ
+          fields:
+            basedate: ${{datetime(year=2000, month=1, day=1)}}
+            dateplus: ${{basedate + "XYZZY"}}
+        """
+        with pytest.raises(exc.DataGenValueError) as e:
+            generate_data(StringIO(yaml), plugin_options={"snowfakery_version": 3})
+        assert "dateplus" in str(e.value)

tests/test_faker.py

@@ -123,27 +123,33 @@ def test_months_past(self, write_row_mock):
         assert (date.today() - the_date).days > 80
         assert (date.today() - the_date).days < 130
 
-    def test_date_times(self, generated_rows):
+    @pytest.mark.parametrize("snowfakery_version", (2, 3))
+    def test_date_times(self, generated_rows, snowfakery_version):
         with open("tests/test_datetimes.yml") as yaml:
-            generate(yaml)
+            generate(yaml, plugin_options={"snowfakery_version": snowfakery_version})
 
         for dt in generated_rows.table_values("Contact", field="EmailBouncedDate"):
             assert "+0" in dt or dt.endswith("Z"), dt
             assert dateparser.isoparse(dt).tzinfo
 
-    def test_hidden_fakers(self):
+    @pytest.mark.parametrize("snowfakery_version", (2, 3))
+    def test_hidden_fakers(self, snowfakery_version):
         yaml = """
         - object: A
           fields:
             date:
               fake: DateTimeThisCentury
         """
         with pytest.raises(exc.DataGenError) as e:
-            generate(StringIO(yaml))
+            generate(
+                StringIO(yaml),
+                plugin_options={"snowfakery_version": snowfakery_version},
+            )
 
         assert e
 
-    def test_bad_tz_param(self):
+    @pytest.mark.parametrize("snowfakery_version", (2, 3))
+    def test_bad_tz_param(self, snowfakery_version):
         yaml = """
         - object: A
           fields:
@@ -152,26 +158,37 @@ def test_bad_tz_param(self):
                 timezone: PST
         """
         with pytest.raises(exc.DataGenError) as e:
-            generate(StringIO(yaml))
+            generate(
+                StringIO(yaml),
+                plugin_options={"snowfakery_version": snowfakery_version},
+            )
 
         assert "timezone" in str(e.value)
         assert "relativedelta" in str(e.value)
 
-    def test_no_timezone(self, generated_rows):
+    @pytest.mark.parametrize("snowfakery_version", (2, 3))
+    def test_no_timezone(self, generated_rows, snowfakery_version):
         yaml = """
         - object: A
           fields:
             date:
               fake.datetime:
                 timezone: False
         """
-        generate(StringIO(yaml))
+        generate(
+            StringIO(yaml),
+            plugin_options={"snowfakery_version": snowfakery_version},
+        )
         date = generated_rows.table_values("A", 0, "date")
         assert dateparser.isoparse(date).tzinfo is None
 
-    def test_relative_dates(self, generated_rows):
+    @pytest.mark.parametrize("snowfakery_version", (2, 3))
+    def test_relative_dates(self, generated_rows, snowfakery_version):
         with open("tests/test_relative_dates.yml") as f:
-            generate(f)
+            generate(
+                f,
+                plugin_options={"snowfakery_version": snowfakery_version},
+            )
         now = datetime.now(timezone.utc)
         # there is a miniscule chance that FutureDateTime picks a DateTime 1 second in the future
         # and then by the time we get here it isn't the future anymore. We'll see if it ever