Progress on #91 (#100)

* Combining additional #88 test with #88 fix and #91

Disabling the Jinja cache results in the unique name generator state disappearing from the global Jinja context:

    >       return _UniqueNameGenerator.ensure_generator_in_globals(env.globals)('c', adj_base_token, '_', '_')
    E       TypeError: 'NoneType' object is not callable
    .tox/py35-test/lib/python3.5/site-packages/nunavut/lang/c.py:527: TypeError

This issue affects all languages. The following test modification indicates that the state gets replaced with None:

    ------------------------- src/nunavut/lang/__init__.py -------------------------
    index bd6524c..135a29a 100644
    @@ -226,7 +226,7 @@ class _UniqueNameGenerator:
        def ensure_generator_in_globals(cls, environment_globals: typing.Dict[str, typing.Any]) -> '_UniqueNameGenerator':
            from .. import TypeLocalGlobalKey

    -        if TypeLocalGlobalKey not in environment_globals:
    +        if environment_globals.get(TypeLocalGlobalKey) is None:
                environment_globals[TypeLocalGlobalKey] = cls()
            return typing.cast('_UniqueNameGenerator', environment_globals[TypeLocalGlobalKey])

Am stuck.

Progress on #91

1) Fixing a problem with unique identifier generation (Issue #88)
2) Generating C++ data structures (Issue #91)

Still TODO for Issue #91 is serialization for types.

* A new way to generate unique ids in a template

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "python.pythonPath": ".pyenv/bin/python",
+    "python.pythonPath": ".tox/local/bin/python",
     "restructuredtext.sphinxBuildPath": "sphinx-build",
     "restructuredtext.confPath": "${workspaceFolder}",
     "restructuredtext.linter.extraArgs": [
@@ -56,6 +56,7 @@
         "getenv",
         "ifdef",
         "ifndef",
+        "inout",
         "isfunction",
         "isroutine",
         "itertools",
@@ -90,5 +91,6 @@
         "wchar",
         "xargs",
         "yamlfy"
-    ]
+    ],
+    "cmake.configureOnOpen": false
 }

CONTRIBUTING.rst

@@ -24,19 +24,17 @@ your dev environment setup.
 Tools
 ************************************************
 
-virtualenv
+tox -e local (virtualenv)
 ================================================
 
-I highly recommend using a virtual environment when doing python development. It'll save you hours
+I highly recommend using the local tox environment when doing python development. It'll save you hours
 of lost productivity the first time it keeps you from pulling in an unexpected dependency from your
-global python environment. You can install virtualenv from brew on osx or apt-get on linux. I'd
+global python environment. You can install tox from brew on osx or apt-get on linux. I'd
 recommend the following environment for vscode::
 
     git submodule update --init --recursive
-    virtualenv -p python3.7 .pyenv
-    source .pyenv/bin/activate
-    pip install -r requirements.txt
-    pip -e install .
+    tox -e local
+    source .tox/local/bin/activate
 
 
 Visual Studio Code
@@ -46,16 +44,14 @@ To use vscode you'll need:
 
 1. vscode
 2. install vscode commandline (`Shell Command: Install`)
-3. virtualenv
+3. tox
 
 Do::
 
     cd path/to/nunavut
     git submodule update --init --recursive
-    virtualenv -p python3.7 .pyenv
-    source .pyenv/bin/activate
-    pip install -r requirements.txt
-    pip -e install .
+    tox -e local
+    source .tox/local/bin/activate
     code .
 
 Then install recommended extensions.

src/conftest.py

@@ -9,6 +9,7 @@
 import typing
 from doctest import ELLIPSIS
 from fnmatch import fnmatch
+from unittest.mock import MagicMock
 
 import pytest
 from sybil import Sybil
@@ -63,6 +64,12 @@ def _make_filter_test_template(filter: typing.Callable,
         else:
             e.filters[filter_name] = filter
 
+        if hasattr(filter, 'contextfilter') and getattr(filter, 'contextfilter'):
+            context = MagicMock()
+            e.filters[filter_name] = functools.partial(filter, context)
+        else:
+            e.filters[filter_name] = filter
+
         if globals is not None:
             e.globals.update(globals)
         rendered = str(e.get_template('test').render())

src/nunavut/__init__.py

@@ -83,12 +83,6 @@
 instead of its current value you will be insulated from this.
 """
 
-TypeLocalGlobalKey = '_nv_template_local_global'
-"""
-A key used in template environments to key storage that is erased before
-processing each new type.
-"""
-
 # +---------------------------------------------------------------------------+
 
 
@@ -429,3 +423,121 @@ def get_or_make_namespace(full_namespace: str) -> typing.Tuple[Namespace, bool]:
 
     # The empty namespace
     return get_or_make_namespace('')[0]
+
+# +---------------------------------------------------------------------------+
+
+
+class Dependencies:
+    """
+    Data structure that contains a set of composite types and annotations (bool flags)
+    which constitute a set of dependencies for a set of DSDL objects.
+    """
+
+    def __init__(self) -> None:
+        self.composite_types = set()  # type: typing.Set[pydsdl.CompositeType]
+        self.uses_integer = False
+        self.uses_float = False
+        self.uses_variable_length_array = False
+        self.uses_array = False
+        self.uses_bool = False
+
+
+class DependencyBuilder:
+    """
+    Given a list of DSDL types this object builds a set of types that the given types use.
+
+    .. invisible-code-block: python
+
+        import pydsdl
+        from unittest.mock import MagicMock
+        from nunavut import DependencyBuilder
+
+        my_dependant_type_l2 = MagicMock(spec=pydsdl.CompositeType)
+        my_dependant_type_l2.parent_service = False
+        my_dependant_type_l2.attributes = []
+
+        my_dependant_type_l1 = MagicMock(spec=pydsdl.CompositeType)
+        my_dependant_type_l1.parent_service = False
+        my_dependant_type_l1.attributes = [MagicMock(data_type=my_dependant_type_l2)]
+
+        my_top_level_type = MagicMock(spec=pydsdl.CompositeType)
+        my_top_level_type.parent_service = False
+        my_top_level_type.attributes = [MagicMock(data_type=my_dependant_type_l1)]
+
+        direct_dependencies = DependencyBuilder(my_top_level_type).direct()
+
+        assert len(direct_dependencies.composite_types) == 1
+        assert my_dependant_type_l1 in direct_dependencies.composite_types
+
+        transitive_dependencies = DependencyBuilder(my_top_level_type).transitive()
+
+        assert len(transitive_dependencies.composite_types) == 2
+        assert my_dependant_type_l1 in transitive_dependencies.composite_types
+        assert my_dependant_type_l2 in transitive_dependencies.composite_types
+
+    :param dependant_types: A list of types to build dependencies for.
+    :type dependant_types: typing.Iterable[pydsdl.Any]
+    """
+
+    def __init__(self, *dependant_types: pydsdl.Any):
+        self._dependent_types = dependant_types
+
+    def transitive(self) -> Dependencies:
+        """
+        Build a set of all transitive dependencies for the dependent types
+        set for this builder.
+        """
+        return self._build_dependency_list(self._dependent_types, True)
+
+    def direct(self) -> Dependencies:
+        """
+        Build a set of all first-order dependencies for the dependent types
+        set for this builder.
+        """
+        return self._build_dependency_list(self._dependent_types, False)
+
+    # +-----------------------------------------------------------------------+
+    # | PRIVATE
+    # +-----------------------------------------------------------------------+
+
+    @classmethod
+    def _build_dependency_list(cls, dependant_types: typing.Iterable[pydsdl.CompositeType], transitive: bool) \
+            -> Dependencies:
+        results = Dependencies()
+        for dependant in dependant_types:
+            cls._extract_dependent_types(cls._extract_data_types(dependant), transitive, results)
+        return results
+
+    @classmethod
+    def _extract_data_types(cls, t: pydsdl.CompositeType) -> typing.List[pydsdl.SerializableType]:
+        # Make a list of all attributes defined by this type
+        if isinstance(t, pydsdl.ServiceType):
+            return [attr.data_type for attr in t.request_type.attributes] + \
+                [attr.data_type for attr in t.response_type.attributes]
+        else:
+            return [attr.data_type for attr in t.attributes]
+
+    @classmethod
+    def _extract_dependent_types(cls,
+                                 dependant_types: typing.Iterable[pydsdl.Any],
+                                 transitive: bool,
+                                 inout_dependencies: Dependencies) -> None:
+        for dt in dependant_types:
+            if isinstance(dt, pydsdl.CompositeType):
+                if dt not in inout_dependencies.composite_types:
+                    inout_dependencies.composite_types.add(dt)
+                    if transitive:
+                        cls._extract_dependent_types(cls._extract_data_types(dt), transitive, inout_dependencies)
+            elif isinstance(dt, pydsdl.ArrayType):
+                if isinstance(dt, pydsdl.VariableLengthArrayType):
+                    inout_dependencies.uses_variable_length_array = True
+                else:
+                    inout_dependencies.uses_array = True
+
+                cls._extract_dependent_types([dt.element_type], transitive, inout_dependencies)
+            elif isinstance(dt, pydsdl.IntegerType):
+                inout_dependencies.uses_integer = True
+            elif isinstance(dt, pydsdl.FloatType):
+                inout_dependencies.uses_float = True
+            elif isinstance(dt, pydsdl.BooleanType):
+                inout_dependencies.uses_bool = True

src/nunavut/jinja/__init__.py

@@ -310,6 +310,37 @@ def is_serializable(value: pydsdl.Any) -> bool:
         """  # noqa: E501
         return isinstance(value, pydsdl.SerializableType)
 
+    @staticmethod
+    def is_None(value: typing.Any) -> bool:
+        """
+        Tests if a value is ``None``
+
+        .. invisible-code-block: python
+
+            from nunavut.jinja import Generator
+            assert Generator.is_None(None) is True
+            assert Generator.is_None(1) is False
+
+        """
+        return (value is None)
+
+    @staticmethod
+    def is_padding(value: pydsdl.Field) -> bool:
+        """
+        Tests if a value is a padding field.
+
+        .. invisible-code-block: python
+
+            from nunavut.jinja import Generator
+            from unittest.mock import MagicMock
+            import pydsdl
+
+            assert Generator.is_padding(MagicMock(spec=pydsdl.PaddingField)) is True
+            assert Generator.is_padding(MagicMock(spec=pydsdl.Field)) is False
+
+        """
+        return isinstance(value, pydsdl.PaddingField)
+
     # +-----------------------------------------------------------------------+
 
     def __init__(self,
@@ -375,7 +406,8 @@ def __init__(self,
                                 keep_trailing_newline=True,
                                 lstrip_blocks=lstrip_blocks,
                                 trim_blocks=trim_blocks,
-                                auto_reload=False)
+                                auto_reload=False,
+                                cache_size=0)
 
         self._add_language_support()
 
@@ -553,10 +585,10 @@ def _generate_type_real(self,
         Logic that should run from _generate_type iff is_dryrun is False.
         """
 
-        from .. import TypeLocalGlobalKey
+        from ..lang import _UniqueNameGenerator
 
         # reset the name generator state for this type
-        self._env.globals[TypeLocalGlobalKey] = None
+        _UniqueNameGenerator.reset()
 
         # Predetermine the post processor types.
         line_pps = []  # type: typing.List['nunavut.postprocessors.LinePostProcessor']

src/nunavut/lang/__init__.py

@@ -8,10 +8,10 @@
 This package contains modules that provide specific support for generating
 source for various languages using templates.
 """
+import importlib
 import inspect
-import typing
 import logging
-import importlib
+import typing
 
 logger = logging.getLogger(__name__)
 
@@ -218,24 +218,26 @@ def get_supported_languages(self) -> typing.Dict[str, Language]:
 class _UniqueNameGenerator:
     """
     Functor used by template filters to obtain a unique name within a given template.
-    This should be made available as a private global "_unique_name_generator" within
-    each template and should be reset after completing generation of a type.
+    This should be made available as a private global within each template.
     """
-
-    @classmethod
-    def ensure_generator_in_globals(cls, environment_globals: typing.Dict[str, typing.Any]) -> '_UniqueNameGenerator':
-        from .. import TypeLocalGlobalKey
-
-        if TypeLocalGlobalKey not in environment_globals:
-            environment_globals[TypeLocalGlobalKey] = cls()
-        return typing.cast('_UniqueNameGenerator', environment_globals[TypeLocalGlobalKey])
+    _singleton = None  # type: typing.Optional['_UniqueNameGenerator']
 
     def __init__(self) -> None:
         self._index_map = {}  # type: typing.Dict[str, typing.Dict[str, int]]
 
+    @classmethod
+    def reset(cls) -> None:
+        cls._singleton = cls()
+
+    @classmethod
+    def get_instance(cls) -> '_UniqueNameGenerator':
+        if cls._singleton is None:
+            raise RuntimeError('No _UniqueNameGenerator has been created. Please use reset to create.')
+        return cls._singleton
+
     def __call__(self, key: str, base_token: str, prefix: str, suffix: str) -> str:
         """
-        Uses a lazy internal index to generate a number unique to a given base_token within a template
+        Uses a global index to generate a number unique to a given base_token within a template
         for a given domain (key).
         """
         try: