Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
* Combining additional OpenCyphal#88 test with OpenCyphal#88 fix and OpenCyphal#91

Disabling the Jinja cache results in the unique name generator state disappearing from the global Jinja context:

    >       return _UniqueNameGenerator.ensure_generator_in_globals(env.globals)('c', adj_base_token, '_', '_')
    E       TypeError: 'NoneType' object is not callable
    .tox/py35-test/lib/python3.5/site-packages/nunavut/lang/c.py:527: TypeError

This issue affects all languages. The following test modification indicates that the state gets replaced with None:

    ------------------------- src/nunavut/lang/__init__.py -------------------------
    index bd6524c..135a29a 100644
    @@ -226,7 +226,7 @@ class _UniqueNameGenerator:
        def ensure_generator_in_globals(cls, environment_globals: typing.Dict[str, typing.Any]) -> '_UniqueNameGenerator':
            from .. import TypeLocalGlobalKey

    -        if TypeLocalGlobalKey not in environment_globals:
    +        if environment_globals.get(TypeLocalGlobalKey) is None:
                environment_globals[TypeLocalGlobalKey] = cls()
            return typing.cast('_UniqueNameGenerator', environment_globals[TypeLocalGlobalKey])

Am stuck.

Progress on OpenCyphal#91

1) Fixing a problem with unique identifier generation (Issue OpenCyphal#88)
2) Generating C++ data structures (Issue OpenCyphal#91)

Still TODO for Issue OpenCyphal#91 is serialization for types.

* A new way to generate unique ids in a template
  • Loading branch information
thirtytwobits authored Nov 16, 2019
1 parent 5ab9cb7 commit 87daa4c
Show file tree
Hide file tree
Showing 23 changed files with 738 additions and 273 deletions.
6 changes: 4 additions & 2 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"python.pythonPath": ".pyenv/bin/python",
"python.pythonPath": ".tox/local/bin/python",
"restructuredtext.sphinxBuildPath": "sphinx-build",
"restructuredtext.confPath": "${workspaceFolder}",
"restructuredtext.linter.extraArgs": [
Expand Down Expand Up @@ -56,6 +56,7 @@
"getenv",
"ifdef",
"ifndef",
"inout",
"isfunction",
"isroutine",
"itertools",
Expand Down Expand Up @@ -90,5 +91,6 @@
"wchar",
"xargs",
"yamlfy"
]
],
"cmake.configureOnOpen": false
}
20 changes: 8 additions & 12 deletions CONTRIBUTING.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,17 @@ your dev environment setup.
Tools
************************************************

virtualenv
tox -e local (virtualenv)
================================================

I highly recommend using a virtual environment when doing python development. It'll save you hours
I highly recommend using the local tox environment when doing python development. It'll save you hours
of lost productivity the first time it keeps you from pulling in an unexpected dependency from your
global python environment. You can install virtualenv from brew on osx or apt-get on linux. I'd
global python environment. You can install tox from brew on osx or apt-get on linux. I'd
recommend the following environment for vscode::

git submodule update --init --recursive
virtualenv -p python3.7 .pyenv
source .pyenv/bin/activate
pip install -r requirements.txt
pip -e install .
tox -e local
source .tox/local/bin/activate


Visual Studio Code
Expand All @@ -46,16 +44,14 @@ To use vscode you'll need:

1. vscode
2. install vscode commandline (`Shell Command: Install`)
3. virtualenv
3. tox

Do::

cd path/to/nunavut
git submodule update --init --recursive
virtualenv -p python3.7 .pyenv
source .pyenv/bin/activate
pip install -r requirements.txt
pip -e install .
tox -e local
source .tox/local/bin/activate
code .

Then install recommended extensions.
Expand Down
48 changes: 0 additions & 48 deletions requirements.txt

This file was deleted.

7 changes: 7 additions & 0 deletions src/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import typing
from doctest import ELLIPSIS
from fnmatch import fnmatch
from unittest.mock import MagicMock

import pytest
from sybil import Sybil
Expand Down Expand Up @@ -63,6 +64,12 @@ def _make_filter_test_template(filter: typing.Callable,
else:
e.filters[filter_name] = filter

if hasattr(filter, 'contextfilter') and getattr(filter, 'contextfilter'):
context = MagicMock()
e.filters[filter_name] = functools.partial(filter, context)
else:
e.filters[filter_name] = filter

if globals is not None:
e.globals.update(globals)
rendered = str(e.get_template('test').render())
Expand Down
124 changes: 118 additions & 6 deletions src/nunavut/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,6 @@
instead of its current value you will be insulated from this.
"""

TypeLocalGlobalKey = '_nv_template_local_global'
"""
A key used in template environments to key storage that is erased before
processing each new type.
"""

# +---------------------------------------------------------------------------+


Expand Down Expand Up @@ -429,3 +423,121 @@ def get_or_make_namespace(full_namespace: str) -> typing.Tuple[Namespace, bool]:

# The empty namespace
return get_or_make_namespace('')[0]

# +---------------------------------------------------------------------------+


class Dependencies:
"""
Data structure that contains a set of composite types and annotations (bool flags)
which constitute a set of dependencies for a set of DSDL objects.
"""

def __init__(self) -> None:
self.composite_types = set() # type: typing.Set[pydsdl.CompositeType]
self.uses_integer = False
self.uses_float = False
self.uses_variable_length_array = False
self.uses_array = False
self.uses_bool = False


class DependencyBuilder:
"""
Given a list of DSDL types this object builds a set of types that the given types use.
.. invisible-code-block: python
import pydsdl
from unittest.mock import MagicMock
from nunavut import DependencyBuilder
my_dependant_type_l2 = MagicMock(spec=pydsdl.CompositeType)
my_dependant_type_l2.parent_service = False
my_dependant_type_l2.attributes = []
my_dependant_type_l1 = MagicMock(spec=pydsdl.CompositeType)
my_dependant_type_l1.parent_service = False
my_dependant_type_l1.attributes = [MagicMock(data_type=my_dependant_type_l2)]
my_top_level_type = MagicMock(spec=pydsdl.CompositeType)
my_top_level_type.parent_service = False
my_top_level_type.attributes = [MagicMock(data_type=my_dependant_type_l1)]
direct_dependencies = DependencyBuilder(my_top_level_type).direct()
assert len(direct_dependencies.composite_types) == 1
assert my_dependant_type_l1 in direct_dependencies.composite_types
transitive_dependencies = DependencyBuilder(my_top_level_type).transitive()
assert len(transitive_dependencies.composite_types) == 2
assert my_dependant_type_l1 in transitive_dependencies.composite_types
assert my_dependant_type_l2 in transitive_dependencies.composite_types
:param dependant_types: A list of types to build dependencies for.
:type dependant_types: typing.Iterable[pydsdl.Any]
"""

def __init__(self, *dependant_types: pydsdl.Any):
self._dependent_types = dependant_types

def transitive(self) -> Dependencies:
"""
Build a set of all transitive dependencies for the dependent types
set for this builder.
"""
return self._build_dependency_list(self._dependent_types, True)

def direct(self) -> Dependencies:
"""
Build a set of all first-order dependencies for the dependent types
set for this builder.
"""
return self._build_dependency_list(self._dependent_types, False)

# +-----------------------------------------------------------------------+
# | PRIVATE
# +-----------------------------------------------------------------------+

@classmethod
def _build_dependency_list(cls, dependant_types: typing.Iterable[pydsdl.CompositeType], transitive: bool) \
-> Dependencies:
results = Dependencies()
for dependant in dependant_types:
cls._extract_dependent_types(cls._extract_data_types(dependant), transitive, results)
return results

@classmethod
def _extract_data_types(cls, t: pydsdl.CompositeType) -> typing.List[pydsdl.SerializableType]:
# Make a list of all attributes defined by this type
if isinstance(t, pydsdl.ServiceType):
return [attr.data_type for attr in t.request_type.attributes] + \
[attr.data_type for attr in t.response_type.attributes]
else:
return [attr.data_type for attr in t.attributes]

@classmethod
def _extract_dependent_types(cls,
dependant_types: typing.Iterable[pydsdl.Any],
transitive: bool,
inout_dependencies: Dependencies) -> None:
for dt in dependant_types:
if isinstance(dt, pydsdl.CompositeType):
if dt not in inout_dependencies.composite_types:
inout_dependencies.composite_types.add(dt)
if transitive:
cls._extract_dependent_types(cls._extract_data_types(dt), transitive, inout_dependencies)
elif isinstance(dt, pydsdl.ArrayType):
if isinstance(dt, pydsdl.VariableLengthArrayType):
inout_dependencies.uses_variable_length_array = True
else:
inout_dependencies.uses_array = True

cls._extract_dependent_types([dt.element_type], transitive, inout_dependencies)
elif isinstance(dt, pydsdl.IntegerType):
inout_dependencies.uses_integer = True
elif isinstance(dt, pydsdl.FloatType):
inout_dependencies.uses_float = True
elif isinstance(dt, pydsdl.BooleanType):
inout_dependencies.uses_bool = True
38 changes: 35 additions & 3 deletions src/nunavut/jinja/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,37 @@ def is_serializable(value: pydsdl.Any) -> bool:
""" # noqa: E501
return isinstance(value, pydsdl.SerializableType)

@staticmethod
def is_None(value: typing.Any) -> bool:
"""
Tests if a value is ``None``
.. invisible-code-block: python
from nunavut.jinja import Generator
assert Generator.is_None(None) is True
assert Generator.is_None(1) is False
"""
return (value is None)

@staticmethod
def is_padding(value: pydsdl.Field) -> bool:
"""
Tests if a value is a padding field.
.. invisible-code-block: python
from nunavut.jinja import Generator
from unittest.mock import MagicMock
import pydsdl
assert Generator.is_padding(MagicMock(spec=pydsdl.PaddingField)) is True
assert Generator.is_padding(MagicMock(spec=pydsdl.Field)) is False
"""
return isinstance(value, pydsdl.PaddingField)

# +-----------------------------------------------------------------------+

def __init__(self,
Expand Down Expand Up @@ -375,7 +406,8 @@ def __init__(self,
keep_trailing_newline=True,
lstrip_blocks=lstrip_blocks,
trim_blocks=trim_blocks,
auto_reload=False)
auto_reload=False,
cache_size=0)

self._add_language_support()

Expand Down Expand Up @@ -553,10 +585,10 @@ def _generate_type_real(self,
Logic that should run from _generate_type iff is_dryrun is False.
"""

from .. import TypeLocalGlobalKey
from ..lang import _UniqueNameGenerator

# reset the name generator state for this type
self._env.globals[TypeLocalGlobalKey] = None
_UniqueNameGenerator.reset()

# Predetermine the post processor types.
line_pps = [] # type: typing.List['nunavut.postprocessors.LinePostProcessor']
Expand Down
28 changes: 15 additions & 13 deletions src/nunavut/lang/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
This package contains modules that provide specific support for generating
source for various languages using templates.
"""
import importlib
import inspect
import typing
import logging
import importlib
import typing

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -218,24 +218,26 @@ def get_supported_languages(self) -> typing.Dict[str, Language]:
class _UniqueNameGenerator:
"""
Functor used by template filters to obtain a unique name within a given template.
This should be made available as a private global "_unique_name_generator" within
each template and should be reset after completing generation of a type.
This should be made available as a private global within each template.
"""

@classmethod
def ensure_generator_in_globals(cls, environment_globals: typing.Dict[str, typing.Any]) -> '_UniqueNameGenerator':
from .. import TypeLocalGlobalKey

if TypeLocalGlobalKey not in environment_globals:
environment_globals[TypeLocalGlobalKey] = cls()
return typing.cast('_UniqueNameGenerator', environment_globals[TypeLocalGlobalKey])
_singleton = None # type: typing.Optional['_UniqueNameGenerator']

def __init__(self) -> None:
self._index_map = {} # type: typing.Dict[str, typing.Dict[str, int]]

@classmethod
def reset(cls) -> None:
cls._singleton = cls()

@classmethod
def get_instance(cls) -> '_UniqueNameGenerator':
if cls._singleton is None:
raise RuntimeError('No _UniqueNameGenerator has been created. Please use reset to create.')
return cls._singleton

def __call__(self, key: str, base_token: str, prefix: str, suffix: str) -> str:
"""
Uses a lazy internal index to generate a number unique to a given base_token within a template
Uses a global index to generate a number unique to a given base_token within a template
for a given domain (key).
"""
try:
Expand Down
Loading

0 comments on commit 87daa4c

Please sign in to comment.