Skip to content

Commit

Permalink
do not autogenerate classes when loading namespace (#555)
Browse files Browse the repository at this point in the history 
* do not autogenerate  classes when loading namespace

* Improve class generation across namespace dependencies (#557)

* Check dependent types, gen classes for them, and link them

* Add comment

* Update src/hdmf/build/manager.py

Co-authored-by: Andrew Tritt <[email protected]>

* add tests for convoluted extension scenario

* generate class if necessary

* make namespace an optional argument

* satisfy flake8

Co-authored-by: Ryan Ly <[email protected]>
  • Loading branch information
@ajtritt @rly
ajtritt and rly authored Apr 9, 2021
1 parent 07b9b95 commit fb37665
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 65 deletions.
9 changes: 2 additions & 7 deletions src/hdmf/build/classgenerator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,16 +145,11 @@ def _get_container_type(cls, type_name, type_map):
"""Search all namespaces for the container class associated with the given data type.
Raises TypeDoesNotExistError if type is not found in any namespace.
"""
container_type = None
for val in type_map.container_types.values():
# NOTE that the type_name may appear in multiple namespaces based on how they were resolved
# but the same type_name should point to the same class
container_type = val.get(type_name)
if container_type is not None:
return container_type
container_type = type_map.get_container_cls(type_name)
if container_type is None: # pragma: no cover
# this should never happen after hdmf#322
raise TypeDoesNotExistError("Type '%s' does not exist." % type_name)
return container_type

@classmethod
def _get_type(cls, spec, type_map):
Expand Down
61 changes: 41 additions & 20 deletions src/hdmf/build/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,38 +474,56 @@ def load_namespaces(self, **kwargs):
for new_ns, ns_deps in deps.items():
for src_ns, types in ns_deps.items():
for dt in types:
container_cls = self.get_container_cls(src_ns, dt)
container_cls = self.get_container_cls(dt, src_ns, autogen=False)
if container_cls is None:
container_cls = TypeSource(src_ns, dt)
self.register_container_type(new_ns, dt, container_cls)
return deps

@docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"},
{"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
@docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
{"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None},
{"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True},
returns='the class for the given namespace and data_type', rtype=type)
def get_container_cls(self, **kwargs):
"""Get the container class from data type specification.
If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically
created and returned.
"""
namespace, data_type = getargs('namespace', 'data_type', kwargs)
namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs)

# namespace is unknown, so look it up
if namespace is None:
for key, val in self.__container_types.items():
# NOTE that the type_name may appear in multiple namespaces based on how they were resolved
# but the same type_name should point to the same class
if data_type in val:
namespace = key
break

cls = self.__get_container_cls(namespace, data_type)
if cls is None: # dynamically generate a class
if cls is None and autogen: # dynamically generate a class
spec = self.__ns_catalog.get_spec(namespace, data_type)
if isinstance(spec, GroupSpec):
self.__resolve_child_container_classes(spec, namespace)
self.__check_dependent_types(spec, namespace)
parent_cls = self.__get_parent_cls(namespace, data_type, spec)
attr_names = self.__default_mapper_cls.get_attr_names(spec)
cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self)
self.register_container_type(namespace, data_type, cls)
return cls

def __resolve_child_container_classes(self, spec, namespace):
for child_spec in (spec.groups + spec.datasets):
if child_spec.data_type_inc is not None:
self.get_container_cls(namespace, child_spec.data_type_inc)
elif child_spec.data_type_def is not None:
self.get_container_cls(namespace, child_spec.data_type_def)
def __check_dependent_types(self, spec, namespace):
"""Ensure that classes for all types used by this type exist and generate them if not.
"""
if spec.data_type_inc is not None:
self.get_container_cls(spec.data_type_inc, namespace)
if isinstance(spec, GroupSpec):
for child_spec in (spec.groups + spec.datasets):
if child_spec.data_type_inc is not None:
self.get_container_cls(child_spec.data_type_inc, namespace)
if child_spec.data_type_def is not None:
self.get_container_cls(child_spec.data_type_def, namespace)
for child_spec in spec.links:
if child_spec.target_type is not None:
self.get_container_cls(child_spec.target_type, namespace)

def __get_parent_cls(self, namespace, data_type, spec):
dt_hier = self.__ns_catalog.get_hierarchy(namespace, data_type)
Expand All @@ -527,15 +545,17 @@ def __get_parent_cls(self, namespace, data_type, spec):
return parent_cls

def __get_container_cls(self, namespace, data_type):
"""Get the container class for the namespace, data_type. If the class doesn't exist yet, generate it."""
if namespace not in self.__container_types:
return None
if data_type not in self.__container_types[namespace]:
return None
ret = self.__container_types[namespace][data_type]
if isinstance(ret, TypeSource):
ret = self.__get_container_cls(ret.namespace, ret.data_type)
if ret is not None:
self.register_container_type(namespace, data_type, ret)
if isinstance(ret, TypeSource): # data_type is a dependency from ret.namespace
cls = self.get_container_cls(ret.data_type, ret.namespace) # get class / generate class
# register the same class into this namespace (replaces TypeSource)
self.register_container_type(namespace, data_type, cls)
ret = cls
return ret

@docval({'name': 'obj', 'type': (GroupBuilder, DatasetBuilder, LinkBuilder, GroupSpec, DatasetSpec),
Expand Down Expand Up @@ -591,7 +611,7 @@ def get_cls(self, **kwargs):
namespace = self.get_builder_ns(builder)
if namespace is None:
raise ValueError("No namespace found for builder %s" % builder.path)
return self.get_container_cls(namespace, data_type)
return self.get_container_cls(data_type, namespace)

@docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'},
{'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder),
Expand Down Expand Up @@ -683,8 +703,9 @@ def register_container_type(self, **kwargs):
self.__container_types.setdefault(namespace, dict())
self.__container_types[namespace][data_type] = container_cls
self.__data_types.setdefault(container_cls, (namespace, data_type))
setattr(container_cls, spec.type_key(), data_type)
setattr(container_cls, 'namespace', namespace)
if not isinstance(container_cls, TypeSource):
setattr(container_cls, spec.type_key(), data_type)
setattr(container_cls, 'namespace', namespace)

@docval({"name": "container_cls", "type": type,
"doc": "the AbstractContainer class for which the given ObjectMapper class gets used for"},
Expand Down
20 changes: 10 additions & 10 deletions src/hdmf/common/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,15 @@ def available_namespaces():
raise RuntimeError("Unable to load a TypeMap - no namespace file found")


DynamicTable = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'DynamicTable')
VectorData = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'VectorData')
VectorIndex = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'VectorIndex')
ElementIdentifiers = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'ElementIdentifiers')
DynamicTableRegion = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'DynamicTableRegion')
EnumData = __TYPE_MAP.get_container_cls(EXP_NAMESPACE, 'EnumData')
CSRMatrix = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'CSRMatrix')
ExternalResources = __TYPE_MAP.get_container_cls(EXP_NAMESPACE, 'ExternalResources')
SimpleMultiContainer = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'SimpleMultiContainer')
DynamicTable = __TYPE_MAP.get_container_cls('DynamicTable', CORE_NAMESPACE)
VectorData = __TYPE_MAP.get_container_cls('VectorData', CORE_NAMESPACE)
VectorIndex = __TYPE_MAP.get_container_cls('VectorIndex', CORE_NAMESPACE)
ElementIdentifiers = __TYPE_MAP.get_container_cls('ElementIdentifiers', CORE_NAMESPACE)
DynamicTableRegion = __TYPE_MAP.get_container_cls('DynamicTableRegion', CORE_NAMESPACE)
EnumData = __TYPE_MAP.get_container_cls('EnumData', EXP_NAMESPACE)
CSRMatrix = __TYPE_MAP.get_container_cls('CSRMatrix', CORE_NAMESPACE)
ExternalResources = __TYPE_MAP.get_container_cls('ExternalResources', EXP_NAMESPACE)
SimpleMultiContainer = __TYPE_MAP.get_container_cls('SimpleMultiContainer', CORE_NAMESPACE)


@docval({'name': 'extensions', 'type': (str, TypeMap, list),
Expand Down Expand Up @@ -197,7 +197,7 @@ def get_class(**kwargs):
"""Get the class object of the Container subclass corresponding to a given neurdata_type.
"""
data_type, namespace = getargs('data_type', 'namespace', kwargs)
return __TYPE_MAP.get_container_cls(namespace, data_type)
return __TYPE_MAP.get_container_cls(data_type, namespace)


@docval({'name': 'io', 'type': HDMFIO,
Expand Down
Loading

0 comments on commit fb37665

Please sign in to comment.