From de587bddaaec6aa1491a44a13f3cba623153e8ac Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 10 Mar 2020 17:18:30 -0700 Subject: [PATCH 001/106] Manual rebase of dev-module-composition on top of current master Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0.py | 47 ++++ .../graph_composition_integration_tests1.py | 44 +++ .../graph_composition_integration_tests2.py | 52 ++++ .../graph_composition_integration_tests3.py | 56 ++++ .../graph_composition_integration_tests4.py | 96 +++++++ nemo/backends/pytorch/actions.py | 17 +- nemo/core/__init__.py | 3 + nemo/core/app_state.py | 97 +++++++ nemo/core/neural_factory.py | 11 +- nemo/core/neural_graph.py | 257 ++++++++++++++++++ nemo/core/neural_graph_manager.py | 121 +++++++++ nemo/core/neural_interface.py | 67 +++++ nemo/core/neural_modules.py | 97 ++++--- tests/unit/core/test_app_state.py | 39 +++ 14 files changed, 949 insertions(+), 55 deletions(-) create mode 100644 examples/start_here/graph_composition_integration_tests0.py create mode 100644 examples/start_here/graph_composition_integration_tests1.py create mode 100644 examples/start_here/graph_composition_integration_tests2.py create mode 100644 examples/start_here/graph_composition_integration_tests3.py create mode 100644 examples/start_here/graph_composition_integration_tests4.py create mode 100644 nemo/core/app_state.py create mode 100644 nemo/core/neural_graph.py create mode 100644 nemo/core/neural_graph_manager.py create mode 100644 nemo/core/neural_interface.py create mode 100644 tests/unit/core/test_app_state.py diff --git a/examples/start_here/graph_composition_integration_tests0.py b/examples/start_here/graph_composition_integration_tests0.py new file mode 100644 index 000000000000..6a5729609d43 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests0.py @@ -0,0 +1,47 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +fx = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can build a `default` (implicit) graph." + F" This approach works for applications containing a single graph/" +) + +# This will create a default (implicit) graph: "training". +x, t = dl() +p = fx(x=x) +lss = loss(predictions=p, target=t) + + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py new file mode 100644 index 000000000000..cb941f921d97 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -0,0 +1,44 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +m2 = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info("This example shows how one can build an `explicit` graph.") + +with NeuralGraph(operation_mode=OperationMode.training) as g0: + x, t = dl() + p = m2(x=x) + lss = loss(predictions=p, target=t) + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py new file mode 100644 index 000000000000..ab92b749a56f --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests2.py @@ -0,0 +1,52 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +m2 = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can nest one graph into another - without binding of the input ports." + F" Please note that the nested graph can be used exatly like any other module" + F" By default, all output graph ports are bound, thus `visible` outside." + F" The user will be able to pick pick a subset manually (this simple feature is in my TODO list)." +) + +with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: + x, t = dl() + y = m2(x=x) + +with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g11: + x1, t1, p1 = g1() + lss = loss(predictions=p1, target=t1) + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests3.py b/examples/start_here/graph_composition_integration_tests3.py new file mode 100644 index 000000000000..5aa3e0aead16 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests3.py @@ -0,0 +1,56 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +fx = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can nest one graph into another - with binding of the input ports." + F" Please note that the nested graph can be used exatly like any other module" + F" In particular, note that the input port 'x' of the module `m2` is bound in graph 'g2'" + F" and then set to `x` returned by `dl` in the graph `g3`." +) + +with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: + # Add module to graph and bind it input port 'x'. + y = fx(x=g2) + +# Build the training graph. +with NeuralGraph(operation_mode=OperationMode.training, name="g3") as g3: + # Add modules to graph. + x, t = dl() + # Incorporate modules from existing graph. + p = g2(x=x) + lss = loss(predictions=p, target=t) + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests4.py b/examples/start_here/graph_composition_integration_tests4.py new file mode 100644 index 000000000000..97851396d2b4 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests4.py @@ -0,0 +1,96 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import torch + +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +dl_validation = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +fx = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can nest one graph (representing the our trained model) into" + F" training and validation graphs." +) + +# Build the training graph. +with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: + # Bind the input. + _ = fx(x=trainable_module) + # All outputs will be bound by default. + +# Compose two graphs into final graph. +with NeuralGraph(operation_mode=OperationMode.training, name="training_graph") as training_graph: + # Take outputs from the training DL. + x, t = dl_training() + # Pass them to the trainable module. + p = trainable_module(x=x) + # Pass both of them to loss. + lss = loss(predictions=p, target=t) + +with NeuralGraph(operation_mode=OperationMode.inference, name="validation_graph") as validation_graph: + # Take outputs from the training DL. + x_valid, t_valid = dl_training() + # Pass them to the trainable module. + p_valid = trainable_module(x=x_valid) + # Pass both of them to loss. + loss_e = loss(predictions=p_valid, target=t_valid) + + +# Callbacks to print info to console and Tensorboard. +train_callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') +) + + +def batch_loss_per_batch_callback(tensors, global_vars): + if "batch_loss" not in global_vars.keys(): + global_vars["batch_loss"] = [] + for key, value in tensors.items(): + if key.startswith("loss"): + global_vars["batch_loss"].append(torch.mean(torch.stack(value))) + + +def batch_loss_epoch_finished_callback(global_vars): + epoch_loss = torch.max(torch.tensor(global_vars["batch_loss"])) + print("Evaluation Loss: {0}".format(epoch_loss)) + return dict({"Evaluation Loss": epoch_loss}) + + +eval_callback = nemo.core.EvaluatorCallback( + eval_tensors=[loss_e], + user_iter_callback=batch_loss_per_batch_callback, + user_epochs_done_callback=batch_loss_epoch_finished_callback, + eval_step=100, +) + +# Invoke "train" action. +nf.train( + [lss], + callbacks=[train_callback, eval_callback], + optimization_params={"num_epochs": 3, "lr": 0.0003}, + optimizer="sgd", +) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 6b807cc0776b..d42d5e085573 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -21,7 +21,7 @@ from nemo.backends.pytorch.optimizers import AdamW, Novograd, master_params from nemo.core import DeploymentFormat, DeviceType, NeuralModule, NmTensor from nemo.core.callbacks import ActionCallback, EvaluatorCallback, SimpleLossLoggerCallback -from nemo.core.neural_factory import Actions, ModelMode, Optimization +from nemo.core.neural_factory import Actions, OperationMode, Optimization from nemo.core.neural_types import * from nemo.utils.helpers import get_checkpoint_from_dir @@ -153,6 +153,7 @@ def is_in_degree_zero(node, processed_nodes): while len(hooks_lst) > 0: # take nmtensor from the end of the list nmtensor = hooks_lst.pop() + node = create_node(nmtensor.producer, nmtensor.producer_args) # Store nmtensor as an output of its producer # first make sure all keys are present per output port @@ -164,7 +165,7 @@ def is_in_degree_zero(node, processed_nodes): all_nodes[node][nmtensor.name] = nmtensor processed_nmtensors.add(nmtensor) if nmtensor.producer_args is not None and nmtensor.producer_args != {}: - for _, new_nmtensor in nmtensor.producer_args.items(): + for name, new_nmtensor in nmtensor.producer_args.items(): if new_nmtensor not in processed_nmtensors: # put in the start of list hooks_lst.insert(0, new_nmtensor) @@ -378,7 +379,7 @@ def __initialize_amp( return optimizer def __nm_graph_forward_pass( - self, call_chain, registered_tensors, mode=ModelMode.train, use_cache=False, + self, call_chain, registered_tensors, mode=OperationMode.training, use_cache=False, ): for ind in range(1, len(call_chain)): if use_cache: @@ -405,16 +406,16 @@ def __nm_graph_forward_pass( # else: # pmodule.enable_allreduce() - if mode == ModelMode.train: + if mode == OperationMode.training: # if module.is_trainable(): if isinstance(pmodule, nn.Module): pmodule.train() - elif mode == ModelMode.eval: + elif mode == OperationMode.inference: # if module.is_trainable(): if isinstance(pmodule, nn.Module): pmodule.eval() else: - raise ValueError("Unknown ModelMode") + raise ValueError("Unknown OperationMode") # prepare call signature for `module` call_set = {} for tensor_name, nmtensor in call_args.items(): @@ -569,7 +570,7 @@ def _eval(self, tensors_2_evaluate, callback, step, verbose=False): t.unique_name: d for t, d in zip(call_chain[0][2].values(), tensors) if t is not None } self.__nm_graph_forward_pass( - call_chain=call_chain, registered_tensors=registered_e_tensors, mode=ModelMode.eval, + call_chain=call_chain, registered_tensors=registered_e_tensors, mode=OperationMode.inference, ) if not is_distributed or self.global_rank == 0: @@ -751,7 +752,7 @@ def _infer( self.__nm_graph_forward_pass( call_chain=call_chain, registered_tensors=registered_e_tensors, - mode=ModelMode.eval, + mode=OperationMode.inference, use_cache=use_cache, ) diff --git a/nemo/core/__init__.py b/nemo/core/__init__.py index 90a1ef4995f3..7979d149e549 100644 --- a/nemo/core/__init__.py +++ b/nemo/core/__init__.py @@ -15,7 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from nemo.core.app_state import AppState from nemo.core.callbacks import * from nemo.core.neural_factory import * +from nemo.core.neural_graph import NeuralGraph +from nemo.core.neural_graph_manager import NeuralGraphManager from nemo.core.neural_modules import * from nemo.core.neural_types import * diff --git a/nemo/core/app_state.py b/nemo/core/app_state.py new file mode 100644 index 000000000000..52d9bd565280 --- /dev/null +++ b/nemo/core/app_state.py @@ -0,0 +1,97 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import threading + +# Sadly have to import this to avoid circular dependencies. +import nemo + + +class Singleton(type): + """ Implementation of a generic singleton meta-class. """ + + # List of instances - one per class. + __instances = {} + # Lock used for accessing the instance. + __lock = threading.Lock() + + def __call__(cls, *args, **kwargs): + """ Returns singleton instance.A thread safe implementation. """ + if cls not in cls.__instances: + # Enter critical section. + with cls.__lock: + # Check once again. + if cls not in cls.__instances: + # Create a new object instance - one per class. + cls.__instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + # Return the instance. + return cls.__instances[cls] + + +class AppState(metaclass=Singleton): + """ + Application state stores variables important from the point of view of execution of the NeMo application. + Staring from the most elementary (epoch number, episode number, device used etc.) to the currently + active graph etc. + """ + + def __init__(self, device=None): + """ + Constructor. Initializes global variables. + + Args: + device: main device used for computations [CPU | GPU] (DEFAULT: GPU) + """ + # Had to set it to None in argument to avoid circular import at the class initialization phase. + if device is None: + self._device = nemo.core.DeviceType.GPU + else: + self._device = device + self._neural_graph_manager = nemo.core.NeuralGraphManager() + + @property + def graphs(self): + """ Property returns the graph manager. + + Returns: + List of created graphs + """ + return self._neural_graph_manager + + def register_graph(self, graph): + """ Registers a new graph. """ + self._neural_graph_manager.register_graph(graph) + + @property + def active_graph(self): + """ Property returns the active graph. + + Returns: + Active graph + """ + return self._neural_graph_manager.active_graph + + @active_graph.setter + def active_graph(self, graph): + """ Property sets the active graph. + + Args: + graph: Neural graph object that will become active. + """ + self._neural_graph_manager.active_graph = graph diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index f8354f80e26c..0b8246dbf9b6 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -17,7 +17,7 @@ __all__ = [ 'Backend', - 'ModelMode', + 'OperationMode', 'Optimization', 'DeviceType', 'Actions', @@ -58,11 +58,12 @@ class Backend(Enum): NotSupported = 2 -class ModelMode(Enum): - """Training Mode or Evaluation/Inference""" +class OperationMode(Enum): + """Training or Inference (Evaluation) mode""" - train = 0 - eval = 1 + training = 0 + inference = 1 + both = 2 class Optimization(Enum): diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py new file mode 100644 index 000000000000..3635d5f6c8fc --- /dev/null +++ b/nemo/core/neural_graph.py @@ -0,0 +1,257 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import collections +from typing import Dict, Optional + +import nemo +from nemo.core.neural_interface import NeuralInterface +from nemo.core.neural_types import ( + NeuralPortNameMismatchError, + NeuralPortNmTensorMismatchError, + NeuralType, + NeuralTypeComparisonResult, +) + + +class NeuralGraph(NeuralInterface): + def __init__(self, operation_mode, name=None): + """ + Constructor. Initializes graph variables. + + Args: + operation_mode: Graph operation mode, that will be propagated along modules during graph creation. + [training | eval] + name: Name of the graph (optional) + """ + # Call integrace constructor. + super().__init__() + # Store name and operation mode. + self._operation_mode = operation_mode + if name is None: + # Simply take the name of operation. + self._name = str(self._operation_mode)[14:] + else: + self._name = name + + # Input and output ports - empty for now. + self._bound_input_ports = {} + self._bound_output_ports = {} + self._bound_input_tensors = {} + self._bound_output_tensors = {} + # List of modules of bound inputs - so we will update their output tensors when the "bound" + # input port will be connected. + self._bound_input_module = {} + + # Operations. + self._operation_list = [] + # Register graph. + self._app_state.register_graph(self) + + def __call__(self, **kwargs): + """ + This method "inserts" one existing neural graph into another one. + Also checks if all inputs were provided and properly connects them. + + """ + # print(" Neural Graph {} __call__".format(self._name)) + # Get input and output ports definitions. + input_port_defs = self.input_ports + output_port_defs = self.output_ports + + # TODO: check graph operation mode compatibility. + + # "Copy" all the operations from the previous graph. + for operation in self._operation_list: + self._app_state.active_graph.record_operation(*operation) + + # print(self._operation_list) + + # Iterate through all passed parameters. + for port_name, port_content in kwargs.items(): + # make sure that passed arguments correspond to input port names + if port_name not in input_port_defs.keys(): + raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) + + # Check what was actually passed. + if isinstance(port_content, nemo.core.NeuralGraph): + + # TODO: make sure that port_content == self._app_state.active_graph ?!?! + + # Bind this input port to a neural graph. + port_content.bind_input(port_name, input_port_defs[port_name], self) + + # It is "compatible by definition";), so we don't have to check this port further. + + else: # : port_content is a neural module. + # Compare input port definition with the received definition. + input_port = input_port_defs[port_name] + type_comatibility = input_port.compare(port_content) + if ( + type_comatibility != NeuralTypeComparisonResult.SAME + and type_comatibility != NeuralTypeComparisonResult.GREATER + ): + raise NeuralPortNmTensorMismatchError( + "\n\nIn {0}. \n" + "Port: {1} and a NmTensor it was fed are \n" + "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" + "\n\nType comparison result: {4}".format( + self.__class__.__name__, + port_name, + input_port_defs[port_name], + port_content, + type_comatibility, + ) + ) + # Reaching that point means that we accepted input to a bound port. + # The current graph parsing requires us to update all outputs of + # a module that "accepted" the input. + # Update means changing the original producer_args for the bound port. + producer = self._bound_input_module[port_name] + for _, output_tensor in self._bound_output_tensors.items(): + if output_tensor.producer == producer: + # Set "input port value" to new content - which indicates tensor (and producer) + # that will be used during graph backward traverse. + output_tensor.producer_args[port_name] = port_content + + # TODO CHECK 1: Are we making sure that ALL necessary inputs that were PASSED? + + # Here we will store the results. + results = None + + # This part is different. Now the goal is not to create NEW "tensors", but to return the bound ones! + if len(output_port_defs) == 1: + # Get the name of the ouput port. + out_name = list(output_port_defs)[0] + # Simply pass the bound tensor. + results = self._bound_output_tensors[out_name] + # BUT UPDATE THE inputs to it!! + + # Bind the output ports. + self._app_state.active_graph.bind_outputs(output_port_defs, [results]) + + else: + result = [] + for _, tensor in self._bound_output_tensors.items(): + result.append(tensor) + + # Creating ad-hoc class for returning from module's forward pass. + output_class_name = f'{self.__class__.__name__}Output' + field_names = list(output_port_defs) + result_type = collections.namedtuple(typename=output_class_name, field_names=field_names,) + + # Bind the output ports. + self._app_state.active_graph.bind_outputs(output_port_defs, result) + + # Tie tuple of output tensors with corresponding names. + results = result_type(*result) + + # Return the results. + return results + + @property + def input_ports(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module input ports. + + Returns: + A (dict) of module's input ports names to NeuralTypes mapping + """ + return self._bound_input_ports + + @property + def output_ports(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports + + Returns: + A (dict) of module's output ports names to NeuralTypes mapping + """ + return self._bound_output_ports + + def __enter__(self): + """ Activates given graph as current. """ + # print("Entering graph: ", self._name) + self._app_state.active_graph = self + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + """ Deactivates current graph. """ + # print("Exiting graph: ", self._name) + self._app_state.active_graph = None + # if exc_type: + # print(f'exc_type: {exc_type}') + # print(f'exc_value: {exc_value}') + # print(f'exc_traceback: {exc_traceback}') + + def __str__(self): + """ Prints a nice summary. """ + # TODO: a nice summary. ;) + desc = "`{}` ({}):\n".format(self._name, len(self._operation_list)) + for op in self._operation_list: + desc = desc + " {}\n".format(type(op[0]).__name__) + return desc + + def record_operation(self, module, inputs): + """ + Records the operation (module plus passed inputs) on a list. + """ + self._operation_list.append([module, inputs]) + + def bind_input(self, port_name, port_definition, bound_module): + # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) + # Copy the definition of the port to graph input port definition. + self._bound_input_ports[port_name] = port_definition + + # Indicate that this tensor is missing and has to be provided! + self._bound_input_tensors[port_name] = None + # Additionally, remember the bound module + self._bound_input_module[port_name] = bound_module + + def bind_outputs(self, output_port_defs, output_values): + # print("Binding ALL outputs: defs = `{}`, values = `{}`".format(output_port_defs, output_values)) + + for (output_name, output_definition), output_value in zip(output_port_defs.items(), output_values): + # print( + # "Binding output: key = `{}`: def = `{}`, value = `{}`".format( + # output_name, type(output_definition), type(output_value) + # ) + # ) + # Copy the definition of the port to graph input port definition. + self._bound_output_ports[output_name] = output_definition + + # Bind output tensors. + self._bound_output_tensors[output_name] = output_value + # Additionally, store all output tensors. + # self._all_output_tensors[output_name] = output_value + + def show_bound_inputs(self): + print("bound input ports: ") + for key, value in self._bound_input_ports.items(): + print(" * `{}`: `{}` ({})".format(key, value, type(value))) + + print("bound input tensors: ") + for key, value in self._bound_input_tensors.items(): + print(" * `{}`: `{}` ({})".format(key, value, type(value))) + + def show_bound_outputs(self): + print("bound output ports: ") + for key, value in self._bound_output_ports.items(): + print(" * `{}`: `{}` ({})".format(key, value, type(value))) + + print("bound output tensors: ") + for key, value in self._bound_output_tensors.items(): + print(" * `{}`: `{}` ({})".format(key, value, type(value))) diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph_manager.py new file mode 100644 index 000000000000..ddf9f4e0a08f --- /dev/null +++ b/nemo/core/neural_graph_manager.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# from collections.abc import Mapping + +from nemo.core.neural_factory import OperationMode +from nemo.core.neural_graph import NeuralGraph + + +class NeuralGraphManager(object): + def __init__(self): + """ + Constructor. Initializes the manager. + + Args: + operation_mode: Graph operation mode, that will be propagated along modules during graph creation. + [training | eval] + """ + self._active_graph = None + self._graphs = {} + + def register_graph(self, graph): + """ Registers a new graph. """ + # Create a unigue name. + # Add it to the list. + unique_name = self.__generate_unique_graph_name(graph._name) + self._graphs[unique_name] = graph + + @property + def graphs(self): + """ Property returns the list of graphs. + + Returns: + List of created graphs. + """ + return self._graphs + + def summary(self): + """ Prints a nice summary. """ + # TODO: a nicer summary. ;) + desc = "" + for name, graph in self._graphs.items(): + desc = desc + "`{}`: {}\n".format(name, graph) + return desc + + @property + def active_graph(self): + """ + Property returns the active graph. If there is no active graph, creates a new one. + + Returns: + Active graph + """ + # Create a new graph - training is the default. + if self._active_graph is None: + # Store graph on the list. + new_graph = NeuralGraph(operation_mode=OperationMode.training) + self.register_graph(new_graph) + # Set the newly created graph as active. + self._active_graph = new_graph + + # Return the graph. + return self._active_graph + + @active_graph.setter + def active_graph(self, graph): + """ Property sets the active graph. + + Args: + graph: Neural graph object that will become active. + """ + # Activate the graph. + self._active_graph = graph + + def __generate_unique_graph_name(self, name): + """ Generates a new unique name by adding postfix (number). """ + # Simply return the same name as long as it is unique. + if name not in self._graphs.keys(): + return name + + # Iterate through numbers. + postfix = 1 + new_name = name + str(postfix) + while new_name in self._graphs.keys(): + postfix = postfix + 1 + new_name = name + str(postfix) + return new_name + + def __len__(self): + """ + + :return: Number of created neural graphs. + + """ + return len(self._graphs) + + def __getitem__(self, key): + """ + Value getter function. + + :param key: Graph name. + + :return: Associated graph. + """ + # Retrieve the value. + return self._graphs[key] diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py new file mode 100644 index 000000000000..b646491ba723 --- /dev/null +++ b/nemo/core/neural_interface.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from abc import ABC, abstractmethod +from typing import Dict, Optional + +import nemo +from nemo.core.neural_types import NeuralType + + +class NeuralInterface(ABC): + """ + Abstract class offering interface shared between Neural Module and Neural Graph. + Had to move it to a separate class to: + a) avoid circular imports between Neural Module and Graph. + b) avoid collection of init_params implemented by default in Neural Module. + c) extract only the methods that are shared (NMs have plenty of methods that are not making any sense for + graph, e.g. get_weights, tie_weights, ) + """ + + def __init__(self): + """ Constructor. Set application state. """ + # Get access to app state. + self._app_state = nemo.core.app_state.AppState() + + @property + @abstractmethod + def input_ports(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module input ports + + Returns: + A (dict) of module's input ports names to NeuralTypes mapping + """ + + @property + @abstractmethod + def output_ports(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports + + Returns: + A (dict) of module's output ports names to NeuralTypes mapping + """ + + @abstractmethod + def __call__(self, **kwargs): + """ + This method is used during the construction of a graph for neural type compatibility checking. + Actual implementation lies in Neural Module and Neural Graph classes. + + Returns: + NmTensor object or tuple of NmTensor objects + """ diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index da3ceec72d59..adcfd5ddd9ed 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -20,7 +20,7 @@ import collections import uuid -from abc import ABC, abstractmethod +from abc import abstractmethod from collections import namedtuple from enum import Enum from inspect import getargvalues, getfullargspec, stack @@ -29,6 +29,7 @@ from ruamel.yaml import YAML +import nemo from .neural_types import ( CanNotInferResultNeuralType, NeuralPortNameMismatchError, @@ -39,6 +40,7 @@ ) from nemo import logging from nemo.core import NeuralModuleFactory +from nemo.core.neural_interface import NeuralInterface from nemo.package_info import __version__ as nemo_version from nemo.utils.decorators.deprecated import deprecated @@ -57,11 +59,13 @@ class WeightShareTransform(Enum): ) -class NeuralModule(ABC): +class NeuralModule(NeuralInterface): """Abstract class that every Neural Module must inherit from. """ def __init__(self): + # Call integrace constructor. + super().__init__() # Get default factory. self._factory = NeuralModuleFactory.get_default_factory() @@ -438,55 +442,55 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ + # print(" Neural Module:__call__") # Get input and output ports definitions. input_port_defs = self.input_ports output_port_defs = self.output_ports + # Record the operation (i.e. add a single module) + self._app_state.active_graph.record_operation(self, kwargs.items()) + first_input_nmtensor_type = None input_nmtensors_are_of_same_type = True - for port_name, tgv in kwargs.items(): + # Iterate through all passed parameters. + for port_name, port_content in kwargs.items(): # make sure that passed arguments correspond to input port names if port_name not in input_port_defs.keys(): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) - input_port = input_port_defs[port_name] - type_comatibility = input_port.compare(tgv) - if ( - type_comatibility != NeuralTypeComparisonResult.SAME - and type_comatibility != NeuralTypeComparisonResult.GREATER - ): - raise NeuralPortNmTensorMismatchError( - "\n\nIn {0}. \n" - "Port: {1} and a NmTensor it was fed are \n" - "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" - "\n\nType comparison result: {4}".format( - self.__class__.__name__, port_name, input_port_defs[port_name], tgv, type_comatibility, + # Check what was actually passed. + if isinstance(port_content, nemo.core.NeuralGraph): + # Bind this input port to a neural graph. + + # TODO: make sure that port_content == self._app_state.active_graph ????? + if port_content != self._app_state.active_graph: + raise ConnectionError("Cannot bind ports of one graph with a different graph!") + port_content.bind_input(port_name, input_port_defs[port_name], self) + # It is "compatible by definition";), so we don't have to check this port further. + else: # : port_content is a neural module. + # Compare input port definition with the received definition. + input_port = input_port_defs[port_name] + type_comatibility = input_port.compare(port_content) + if ( + type_comatibility != NeuralTypeComparisonResult.SAME + and type_comatibility != NeuralTypeComparisonResult.GREATER + ): + raise NeuralPortNmTensorMismatchError( + "\n\nIn {0}. \n" + "Port: {1} and a NmTensor it was fed are \n" + "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" + "\n\nType comparison result: {4}".format( + self.__class__.__name__, + port_name, + input_port_defs[port_name], + port_content, + type_comatibility, + ) ) - ) + # TODO CHECK 1: Are we making sure that ALL necessary inputs that were PASSED? - # if first_input_nmtensor_type is None: - # first_input_nmtensor_type = NeuralType(tgv._axis2type) - # else: - # if first_input_nmtensor_type._axis2type is None: - # input_nmtensors_are_of_same_type = True - # else: - # input_nmtensors_are_of_same_type = first_input_nmtensor_type.compare( - # tgv - # ) == NeuralTypeComparisonResult.SAME and len(first_input_nmtensor_type._axis2type) - # if not ( - # type_comatibility == NeuralTypeComparisonResult.SAME - # or type_comatibility == NeuralTypeComparisonResult.GREATER - # ): - # raise NeuralPortNmTensorMismatchError( - # "\n\nIn {0}. \n" - # "Port: {1} and a NmTensor it was fed are \n" - # "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" - # "\n\nType comparison result: {4}".format( - # self.__class__.__name__, port_name, input_port_defs[port_name], tgv, type_comatibility, - # ) - # ) - # if type_comatibility == NeuralTypeComparisonResult.LESS: - # print('Types were raised') + # Here we will store the results. + results = None if len(output_port_defs) == 1: out_name = list(output_port_defs)[0] @@ -498,7 +502,12 @@ def __call__(self, **kwargs): raise CanNotInferResultNeuralType( "Can't infer output neural type. Likely your inputs are of different type." ) - return NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) + # TODO CHECK 2: Why are we returning "something" (having input type) if there SHOULD be NO output? + results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) + + # Bind the output ports. + self._app_state.active_graph.bind_outputs(output_port_defs, [results]) + else: result = [] for out_port, n_type in output_port_defs.items(): @@ -517,10 +526,14 @@ def __call__(self, **kwargs): field_names = list(output_port_defs) result_type = collections.namedtuple(typename=output_class_name, field_names=field_names,) + # Bind the output ports. + self._app_state.active_graph.bind_outputs(output_port_defs, result) + # Tie tuple of output tensors with corresponding names. - result = result_type(*result) + results = result_type(*result) - return result + # Return the results. + return results def __str__(self): return self.__class__.__name__ diff --git a/tests/unit/core/test_app_state.py b/tests/unit/core/test_app_state.py new file mode 100644 index 000000000000..71361003810a --- /dev/null +++ b/tests/unit/core/test_app_state.py @@ -0,0 +1,39 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from unittest import TestCase + +import pytest +from ruamel.yaml import YAML + +from nemo.core import AppState + + +class TestAppState(TestCase): + @pytest.mark.unit + def test_shared_graph(self): + # Create first instance of AppState. + x = AppState() + x.test_value = "ala" + # Create second instance of AppState and test value. + y = AppState() + self.assertEqual(y.test_value, "ala") + # Change second instance and test first one. + y.test_value = "ola" + self.assertEqual(x.test_value, "ola") From 15b3b11926c6a9e551ab93118018a48ba4c29cb2 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 24 Mar 2020 13:25:41 -0700 Subject: [PATCH 002/106] Adding neural graph examples Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 95 +++++++++++++++++++ .../graph_composition_integration_tests1_1.py | 49 ++++++++++ .../graph_composition_integration_tests1_2.py | 54 +++++++++++ 3 files changed, 198 insertions(+) create mode 100644 examples/start_here/graph_composition_integration_tests0_jasper.py create mode 100644 examples/start_here/graph_composition_integration_tests1_1.py create mode 100644 examples/start_here/graph_composition_integration_tests1_2.py diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py new file mode 100644 index 000000000000..042b1665923e --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -0,0 +1,95 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from os.path import expanduser +from functools import partial + +from ruamel.yaml import YAML + +import nemo +from nemo.core import NeuralGraph, OperationMode +import nemo.collections.asr as nemo_asr +from nemo.collections.asr.helpers import monitor_asr_train_progress + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +fx = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can build a Jasper model using the `default` (implicit) graph." + F" This approach works for applications containing a single graph." +) + +# Set paths to "manifests" and model configuration files. +train_manifest="~/TestData/an4_dataset/an4_train.json" +val_manifest="~/TestData/an4_dataset/an4_val.json" +model_config_file="~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" + +yaml = YAML(typ="safe") +with open(expanduser(model_config_file)) as f: + jasper_params = yaml.load(f) +# Get vocabulary. +vocab = jasper_params['labels'] + +# Create neural modules. +data_layer = nemo_asr.AudioToTextDataLayer.import_from_config( + model_config_file, + "AudioToTextDataLayer_train", + overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, +) + +data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.import_from_config( + model_config_file, "AudioToMelSpectrogramPreprocessor" +) + +jasper_encoder = nemo_asr.JasperEncoder.import_from_config(model_config_file, "JasperEncoder") +jasper_decoder = nemo_asr.JasperDecoderForCTC.import_from_config( + model_config_file, "JasperDecoderForCTC", overwrite_params={"num_classes": len(vocab)} +) +ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) +greedy_decoder = nemo_asr.GreedyCTCDecoder() + +# Create the Jasper composite module. +with NeuralGraph(operation_mode=OperationMode.training) as Jasper: + processed_signal, processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. + encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=processed_signal_len) + log_probs = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) + +# Create the "implicit" training graph. +audio_signal, audio_signal_len, transcript, transcript_len = data_layer() +# Use Jasper module as any other neural module. +_, _, _, encoded_len, log_probs = Jasper(input_signal=audio_signal, length=audio_signal_len) +predictions = greedy_decoder(log_probs=log_probs) +loss = ctc_loss(log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len) +tensors_to_evaluate = [loss, predictions, transcript, transcript_len] + +train_callback = nemo.core.SimpleLossLoggerCallback( + tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) +) + +nf.train( + tensors_to_optimize=[loss], + optimizer="novograd", + callbacks=[train_callback], + optimization_params={"num_epochs": 50, "lr": 0.01}, +) diff --git a/examples/start_here/graph_composition_integration_tests1_1.py b/examples/start_here/graph_composition_integration_tests1_1.py new file mode 100644 index 000000000000..8d5cad0672de --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests1_1.py @@ -0,0 +1,49 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +m2 = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info("This example shows how one can build an `explicit` graph." + F"It also shows how to decouple graph instance creation from its activation.") + +# Create the g0 graph. +g0 = NeuralGraph(operation_mode=OperationMode.training) + +# Activate the "g0 graph context" - all operations will be recorded to g0. +with g0: + x, t = dl() + p = m2(x=x) + lss = loss(predictions=p, target=t) + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests1_2.py b/examples/start_here/graph_composition_integration_tests1_2.py new file mode 100644 index 000000000000..f5a5c9fea7bd --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests1_2.py @@ -0,0 +1,54 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import NeuralGraph, OperationMode, AppState + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +m2 = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info("This example shows how one can build an `explicit` graph." + F"It also shows how to activate and deactivate the g0 context `manually`") + +# Create the g0 graph. +g0 = NeuralGraph(operation_mode=OperationMode.training) + +# Activate the "g0 graph context" "manually" - all operations will be recorded to g0. +g0.activate() + +# Define g0 - connections between the modules. +x, t = dl() +p = m2(x=x) +lss = loss(predictions=p, target=t) + +# Deactivate the "g0 graph context" (this is really optional, as long as there are no other operations). +g0.deactivate() + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") From 2756a66bae870e62fd683c01df8bdf0882fb8b4f Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 24 Mar 2020 13:26:57 -0700 Subject: [PATCH 003/106] formatting fixes Signed-off-by: Tomasz Kornuta --- ...raph_composition_integration_tests0_jasper.py | 16 ++++++++-------- .../graph_composition_integration_tests1_1.py | 6 ++++-- .../graph_composition_integration_tests1_2.py | 8 +++++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 042b1665923e..9e7e0025f52b 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -17,15 +17,15 @@ # limitations under the License. # ============================================================================= -from os.path import expanduser from functools import partial +from os.path import expanduser from ruamel.yaml import YAML import nemo -from nemo.core import NeuralGraph, OperationMode import nemo.collections.asr as nemo_asr from nemo.collections.asr.helpers import monitor_asr_train_progress +from nemo.core import NeuralGraph, OperationMode logging = nemo.logging @@ -41,9 +41,9 @@ ) # Set paths to "manifests" and model configuration files. -train_manifest="~/TestData/an4_dataset/an4_train.json" -val_manifest="~/TestData/an4_dataset/an4_val.json" -model_config_file="~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" +train_manifest = "~/TestData/an4_dataset/an4_train.json" +val_manifest = "~/TestData/an4_dataset/an4_val.json" +model_config_file = "~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" yaml = YAML(typ="safe") with open(expanduser(model_config_file)) as f: @@ -71,14 +71,14 @@ # Create the Jasper composite module. with NeuralGraph(operation_mode=OperationMode.training) as Jasper: - processed_signal, processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. + processed_signal, processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=processed_signal_len) - log_probs = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) + log_probs = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) # Create the "implicit" training graph. audio_signal, audio_signal_len, transcript, transcript_len = data_layer() # Use Jasper module as any other neural module. -_, _, _, encoded_len, log_probs = Jasper(input_signal=audio_signal, length=audio_signal_len) +_, _, _, encoded_len, log_probs = Jasper(input_signal=audio_signal, length=audio_signal_len) predictions = greedy_decoder(log_probs=log_probs) loss = ctc_loss(log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len) tensors_to_evaluate = [loss, predictions, transcript, transcript_len] diff --git a/examples/start_here/graph_composition_integration_tests1_1.py b/examples/start_here/graph_composition_integration_tests1_1.py index 8d5cad0672de..e915831faa93 100644 --- a/examples/start_here/graph_composition_integration_tests1_1.py +++ b/examples/start_here/graph_composition_integration_tests1_1.py @@ -28,8 +28,10 @@ m2 = nemo.tutorials.TaylorNet(dim=4) loss = nemo.tutorials.MSELoss() -logging.info("This example shows how one can build an `explicit` graph." - F"It also shows how to decouple graph instance creation from its activation.") +logging.info( + "This example shows how one can build an `explicit` graph." + F"It also shows how to decouple graph instance creation from its activation." +) # Create the g0 graph. g0 = NeuralGraph(operation_mode=OperationMode.training) diff --git a/examples/start_here/graph_composition_integration_tests1_2.py b/examples/start_here/graph_composition_integration_tests1_2.py index f5a5c9fea7bd..cfce01d5f62c 100644 --- a/examples/start_here/graph_composition_integration_tests1_2.py +++ b/examples/start_here/graph_composition_integration_tests1_2.py @@ -18,7 +18,7 @@ # ============================================================================= import nemo -from nemo.core import NeuralGraph, OperationMode, AppState +from nemo.core import AppState, NeuralGraph, OperationMode logging = nemo.logging @@ -28,8 +28,10 @@ m2 = nemo.tutorials.TaylorNet(dim=4) loss = nemo.tutorials.MSELoss() -logging.info("This example shows how one can build an `explicit` graph." - F"It also shows how to activate and deactivate the g0 context `manually`") +logging.info( + "This example shows how one can build an `explicit` graph." + F"It also shows how to activate and deactivate the g0 context `manually`" +) # Create the g0 graph. g0 = NeuralGraph(operation_mode=OperationMode.training) From 5bad9c9d8fe0b23dfde8628450de193b95eed87f Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2020 15:55:04 -0700 Subject: [PATCH 004/106] new proposal Signed-off-by: Jason --- .../graph_composition_integration_tests1_3.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 examples/start_here/graph_composition_integration_tests1_3.py diff --git a/examples/start_here/graph_composition_integration_tests1_3.py b/examples/start_here/graph_composition_integration_tests1_3.py new file mode 100755 index 000000000000..e587e9b06e78 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests1_3.py @@ -0,0 +1,84 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core import AppState, NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +m2 = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can build an `explicit` graph." + F"It also shows how to activate and deactivate the g0 context `manually`" +) + + +def NeuralGraphDecorator(func): + def wrapper(*args, **kwargs): + # Create the g0 graph. + g0 = NeuralGraph(operation_mode=OperationMode.training) + + # Activate the "g0 graph context" "manually" - all operations will be recorded to g0. + g0.activate() + + # Extract input_ports + input_ports = list(args) + for key, value in kwargs.items(): + input_ports.append(value) + + # Run user-defined function + output_ports = func(*args, **kwargs) + + # Record ports + g0.input_ports = input_ports + g0.output_ports = output_ports + + # Deactivate the "g0 graph context" (this is really optional, as long as there are no other operations). + g0.deactive() + + # Return our new compose neural module + return g0 + return wrapper + +@NeuralGraphDecorator +def my_DAG(): + x, t = dl() + p = m2(x=x) + lss = loss(predictions=p, target=t) + return lss + +graph = my_DAG() +lss = graph.output_ports + +## Pros: functions are easy to understand +## Cons: function must return nxTensors, eg cannot create callbacks in them +## need to return relevant tensors + +# SimpleLossLoggerCallback will print loss values to console. +callback = nemo.core.SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") From 0576ce74d734b9608d83a6219b1ac0f4c20d2517 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 Apr 2020 16:08:36 -0700 Subject: [PATCH 005/106] format fix of Jason's example Signed-off-by: Tomasz Kornuta --- examples/start_here/graph_composition_integration_tests1_3.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/start_here/graph_composition_integration_tests1_3.py b/examples/start_here/graph_composition_integration_tests1_3.py index e587e9b06e78..2fbd6d7c9e08 100755 --- a/examples/start_here/graph_composition_integration_tests1_3.py +++ b/examples/start_here/graph_composition_integration_tests1_3.py @@ -59,8 +59,10 @@ def wrapper(*args, **kwargs): # Return our new compose neural module return g0 + return wrapper + @NeuralGraphDecorator def my_DAG(): x, t = dl() @@ -68,6 +70,7 @@ def my_DAG(): lss = loss(predictions=p, target=t) return lss + graph = my_DAG() lss = graph.output_ports From 3f1ac0138abc4b13329f4f4912ea56e07eb8491e Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 Apr 2020 17:40:14 -0700 Subject: [PATCH 006/106] Added modules recording, with retrieval of module name Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests2.py | 3 +- .../graph_composition_integration_tests4.py | 1 - .../graph_composition_integration_tests5.py | 67 +++++++++++++++++++ nemo/core/neural_graph.py | 58 +++++++++++++--- nemo/core/neural_modules.py | 4 +- nemo/utils/__init__.py | 2 + nemo/utils/retrieve_variable_name.py | 37 ++++++++++ 7 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 examples/start_here/graph_composition_integration_tests5.py create mode 100644 nemo/utils/retrieve_variable_name.py diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py index ab92b749a56f..d4de7eb72c05 100644 --- a/examples/start_here/graph_composition_integration_tests2.py +++ b/examples/start_here/graph_composition_integration_tests2.py @@ -29,10 +29,9 @@ loss = nemo.tutorials.MSELoss() logging.info( - "This example shows how one can nest one graph into another - without binding of the input ports." + "This example shows how one can nest one graph into another - with binding of output ports." F" Please note that the nested graph can be used exatly like any other module" F" By default, all output graph ports are bound, thus `visible` outside." - F" The user will be able to pick pick a subset manually (this simple feature is in my TODO list)." ) with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: diff --git a/examples/start_here/graph_composition_integration_tests4.py b/examples/start_here/graph_composition_integration_tests4.py index 97851396d2b4..e4c499143fd3 100644 --- a/examples/start_here/graph_composition_integration_tests4.py +++ b/examples/start_here/graph_composition_integration_tests4.py @@ -56,7 +56,6 @@ x_valid, t_valid = dl_training() # Pass them to the trainable module. p_valid = trainable_module(x=x_valid) - # Pass both of them to loss. loss_e = loss(predictions=p_valid, target=t_valid) diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py new file mode 100644 index 000000000000..6dbf96e364eb --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -0,0 +1,67 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import torch + +import inspect +import nemo +from nemo.core import NeuralGraph, OperationMode + +logging = nemo.logging + +nf = nemo.core.NeuralModuleFactory() +# Instantiate the necessary neural modules. +dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) +fx = nemo.tutorials.TaylorNet(dim=4) +loss = nemo.tutorials.MSELoss() + +logging.info( + "This example shows how one can access modules nested in a graph." +) + +# Build the training graph. +with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: + # Bind the input. + _ = fx(x=trainable_module) + # All outputs will be bound by default. + +# Compose two graphs into final graph. +with NeuralGraph(operation_mode=OperationMode.training, name="training_graph") as training_graph: + # Take outputs from the training DL. + x, t = dl_training() + # Pass them to the trainable module. + p = trainable_module(x=x) + # Pass both of them to loss. + lss = loss(predictions=p, target=t) + +print(trainable_module.list_modules()) + +print(training_graph.list_modules()) + +# Access modules. +dl_training_ref = training_graph["dl_training"] +fx_ref = training_graph["fx"] +loss_ref = training_graph["loss"] + +# Throws an exception. +try: + _ = training_graph["other_module"] +except KeyError as e: + print("Got error: {}".format(e)) + pass \ No newline at end of file diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 3635d5f6c8fc..ac00ecb92235 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -27,9 +27,16 @@ NeuralType, NeuralTypeComparisonResult, ) +from nemo.utils.retrieve_variable_name import retrieve_variable_name class NeuralGraph(NeuralInterface): + """ + Neural Graph class stores dynamically defined graphs of connected Neural Modules. + """ + + + def __init__(self, operation_mode, name=None): """ Constructor. Initializes graph variables. @@ -41,6 +48,7 @@ def __init__(self, operation_mode, name=None): """ # Call integrace constructor. super().__init__() + # Store name and operation mode. self._operation_mode = operation_mode if name is None: @@ -58,8 +66,10 @@ def __init__(self, operation_mode, name=None): # input port will be connected. self._bound_input_module = {} - # Operations. - self._operation_list = [] + # "Modules" - list of modules constituting edges in a given graph. + self._modules = {} + # "Steps": ordered execution of modules in a graph. + self._steps = [] # Register graph. self._app_state.register_graph(self) @@ -77,10 +87,10 @@ def __call__(self, **kwargs): # TODO: check graph operation mode compatibility. # "Copy" all the operations from the previous graph. - for operation in self._operation_list: - self._app_state.active_graph.record_operation(*operation) + for step in self._steps: + self._app_state.active_graph.record_step(*step) - # print(self._operation_list) + # print(self._steps) # Iterate through all passed parameters. for port_name, port_content in kwargs.items(): @@ -200,16 +210,46 @@ def __exit__(self, exc_type, exc_value, exc_traceback): def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) - desc = "`{}` ({}):\n".format(self._name, len(self._operation_list)) - for op in self._operation_list: + desc = "`{}` ({}):\n".format(self._name, len(self._steps)) + for op in self._steps: desc = desc + " {}\n".format(type(op[0]).__name__) return desc - def record_operation(self, module, inputs): + def __getitem__(self, key): + """ Returns module given its name (name of the variable). + + Args: + key: Name of the variable. + """ + if key not in self._modules.keys(): + raise KeyError("Neural Graph doesn't contain a module named {}".format(key)) + return self._modules[key] + + def __len__(self): + return len(self._modules) + + def list_modules(self): + desc = "{} ({}):\n".format(retrieve_variable_name(self), len(self)) + for key, value in self._modules.items(): + desc += " * `{}` ({})\n".format(key, value ) + return desc + + def record_step(self, module, inputs): """ Records the operation (module plus passed inputs) on a list. """ - self._operation_list.append([module, inputs]) + # Get module name. + module_name = retrieve_variable_name(module) + #print("module_name: ", module_name) + + # Check if module with that name already exists. + if module_name in self._modules.keys(): + raise KeyError("Neural Graph already contains a module named {}".format(module_name)) + # Add module to list of modules. + self._modules[module_name] = module + + # Add step. + self._steps.append([module, inputs]) def bind_input(self, port_name, port_definition, bound_module): # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index adcfd5ddd9ed..3f4fb124c9ac 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -447,8 +447,8 @@ def __call__(self, **kwargs): input_port_defs = self.input_ports output_port_defs = self.output_ports - # Record the operation (i.e. add a single module) - self._app_state.active_graph.record_operation(self, kwargs.items()) + # Record the operation (i.e. add a single module). + self._app_state.active_graph.record_step(self, kwargs.items()) first_input_nmtensor_type = None input_nmtensors_are_of_same_type = True diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 255316fef491..30707e573df5 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -24,3 +24,5 @@ from .argparse import NemoArgParser from .exp_logging import ExpManager, get_logger from .helpers import * + +from .retrieve_variable_name import retrieve_variable_name \ No newline at end of file diff --git a/nemo/utils/retrieve_variable_name.py b/nemo/utils/retrieve_variable_name.py new file mode 100644 index 000000000000..5d4d53ea9122 --- /dev/null +++ b/nemo/utils/retrieve_variable_name.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import inspect + +def retrieve_variable_name(var): + """ + Function returns the name of the variable. + Throws the KeyError exception is name was not found. + + Args: + var: Variable. + + Returns: + String representing the variable name. + """ + for fi in reversed(inspect.stack()): + names = [var_name for var_name, var_val in fi.frame.f_locals.items() if var_val is var] + if len(names) > 0: + return names[0] + else: + raise KeyError From a57b34ded2687a147b358f938817f598f986c86a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 7 Apr 2020 08:55:55 -0700 Subject: [PATCH 007/106] keyerror comment Signed-off-by: Tomasz Kornuta --- examples/start_here/graph_composition_integration_tests5.py | 1 - nemo/utils/retrieve_variable_name.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py index 6dbf96e364eb..ef7aa7eef0d2 100644 --- a/examples/start_here/graph_composition_integration_tests5.py +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -19,7 +19,6 @@ import torch -import inspect import nemo from nemo.core import NeuralGraph, OperationMode diff --git a/nemo/utils/retrieve_variable_name.py b/nemo/utils/retrieve_variable_name.py index 5d4d53ea9122..f59d1eca40bd 100644 --- a/nemo/utils/retrieve_variable_name.py +++ b/nemo/utils/retrieve_variable_name.py @@ -34,4 +34,4 @@ def retrieve_variable_name(var): if len(names) > 0: return names[0] else: - raise KeyError + raise KeyError("Cannot retrieve the name of object {} as it is not present on the stack (yet)".format(var)) From fc3deac95e7bdb03d3ddd49003d93a6cd2e3e015 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 7 Apr 2020 12:44:58 -0700 Subject: [PATCH 008/106] Introduced name property of modules Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests5.py | 7 +- nemo/backends/pytorch/nm.py | 18 ++--- nemo/backends/pytorch/tutorials/toys.py | 38 +++++----- nemo/core/neural_graph.py | 69 +++++++++++-------- nemo/core/neural_modules.py | 19 ++--- nemo/utils/__init__.py | 2 - nemo/utils/retrieve_variable_name.py | 37 ---------- 7 files changed, 84 insertions(+), 106 deletions(-) delete mode 100644 nemo/utils/retrieve_variable_name.py diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py index ef7aa7eef0d2..52c4c1a78982 100644 --- a/examples/start_here/graph_composition_integration_tests5.py +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -26,9 +26,10 @@ nf = nemo.core.NeuralModuleFactory() # Instantiate the necessary neural modules. -dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -fx = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() +dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128, name="dl_training") +fx = nemo.tutorials.TaylorNet(dim=4, name="fx") +print(fx.name) +loss = nemo.tutorials.MSELoss(name="loss") logging.info( "This example shows how one can access modules nested in a graph." diff --git a/nemo/backends/pytorch/nm.py b/nemo/backends/pytorch/nm.py index 6e24218c96db..06938a66ba79 100644 --- a/nemo/backends/pytorch/nm.py +++ b/nemo/backends/pytorch/nm.py @@ -34,9 +34,9 @@ def __init__(self): """ - def __init__(self, pretrained_model_name=None): - - NeuralModule.__init__(self) # For NeuralModule API + def __init__(self, pretrained_model_name=None, name=None): + print("TrainableNM: ", name) + NeuralModule.__init__(self, name) # For NeuralModule API nn.Module.__init__(self) # For PyTorch API self._device = get_cuda_device(self.placement) @@ -130,8 +130,8 @@ def num_weights(self): class NonTrainableNM(NeuralModule): - def __init__(self): - NeuralModule.__init__(self) # For NeuralModule API + def __init__(self, name=None): + NeuralModule.__init__(self, name) # For NeuralModule API self._device = get_cuda_device(self.placement) def __call__(self, force_pt=False, *input, **kwargs): @@ -190,8 +190,8 @@ class DataLayerNM(NeuralModule): data_iterator property to return iterator over the dataset. """ - def __init__(self): - NeuralModule.__init__(self) # For NeuralModule API + def __init__(self, name=None): + NeuralModule.__init__(self, name) # For NeuralModule API self._device = get_cuda_device(self.placement) # if 'batch_size' not in kwargs: @@ -325,8 +325,8 @@ class LossNM(NeuralModule): You must implement _loss_function method. """ - def __init__(self): - NeuralModule.__init__(self) # For NeuralModule API + def __init__(self, name=None): + NeuralModule.__init__(self, name) # For NeuralModule API self._device = get_cuda_device(self.placement) def get_weights(self): diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 25fa8bd7c277..b524e2c28a26 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -35,11 +35,16 @@ def output_ports(self): """ return {"y_pred": NeuralType(('B', 'D'), ChannelType())} - def __init__(self, dim): - # Part specific for Neural Modules API: - # (1) call base constructor - # (2) define input and output ports - super().__init__() + def __init__(self, dim, name=None): + """ + Creates TaylorNet object. + + Args: + dim: Number of dimensions (number of terms in Taylor series). + name: Name of the module instance + """ + print("TaylorNet: ", name) + super().__init__(name=name) # And of Neural Modules specific part. Rest is Pytorch code self._dim = dim @@ -78,11 +83,11 @@ def output_ports(self): """ return {"y_pred": NeuralType(('B', 'D'), ChannelType(), optional=True)} - def __init__(self, dim): + def __init__(self, dim,name=None): # Part specific for Neural Modules API: # (1) call base constructor # (2) define input and output ports - super().__init__() + super().__init__(name=name) # And of Neural Modules specific part. Rest is Pytorch code self._dim = dim @@ -133,7 +138,7 @@ def output_ports(self): "y": NeuralType(('B', 'D'), LabelsType()), } - def __init__(self, batch_size, f_name="sin", n=1000, x_lo=-4, x_hi=4): + def __init__(self, batch_size, f_name="sin", n=1000, x_lo=-4, x_hi=4, name=None): """ Creates a datalayer returning (x-y) pairs, with n points from a given range. @@ -143,8 +148,9 @@ def __init__(self, batch_size, f_name="sin", n=1000, x_lo=-4, x_hi=4): n: number of points x_lo: lower boundary along x axis x_hi: higher boundary along x axis + name: Name of the module instance """ - super().__init__() + super().__init__(name=name) # Dicionary with handled functions. handled_funcs = {"sin": t.sin, "cos": t.cos} @@ -200,8 +206,8 @@ def output_ports(self): """ return {"loss": NeuralType(elements_type=LossType())} - def __init__(self): - super().__init__() + def __init__(self, name=None): + super().__init__(name=name) self._criterion = nn.MSELoss() def _loss_function(self, **kwargs): @@ -226,8 +232,8 @@ def output_ports(self): """ return {"loss": NeuralType(elements_type=LossType())} - def __init__(self): - super().__init__() + def __init__(self, name=None): + super().__init__(name=name) self._criterion = nn.L1Loss() def _loss_function(self, **kwargs): @@ -255,10 +261,8 @@ def output_ports(self): """ return {"loss": NeuralType(elements_type=LossType())} - def __init__(self): - # Neural Module API specific - NeuralModule.__init__(self) - # End of Neural Module API specific + def __init__(self, name=None): + super().__init__(name=name) self._criterion = nn.CrossEntropyLoss() # You need to implement this function diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index ac00ecb92235..46735538e03e 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -27,8 +27,6 @@ NeuralType, NeuralTypeComparisonResult, ) -from nemo.utils.retrieve_variable_name import retrieve_variable_name - class NeuralGraph(NeuralInterface): """ @@ -57,19 +55,25 @@ def __init__(self, operation_mode, name=None): else: self._name = name - # Input and output ports - empty for now. + # Input ports and tensors - empty for now. self._bound_input_ports = {} - self._bound_output_ports = {} self._bound_input_tensors = {} - self._bound_output_tensors = {} # List of modules of bound inputs - so we will update their output tensors when the "bound" # input port will be connected. - self._bound_input_module = {} + self._bound_input_modules = {} + + # Output ports and tensors - used in manual binding, empty for now. + self._bound_output_tensors_manual = {} + self._bound_output_ports_manual = {} + # Default output ports and tensors - used in automatic binding, empty for now. + self._bound_output_tensors_default = {} + self._bound_output_ports_default = {} # "Modules" - list of modules constituting edges in a given graph. self._modules = {} # "Steps": ordered execution of modules in a graph. self._steps = [] + # Register graph. self._app_state.register_graph(self) @@ -132,8 +136,8 @@ def __call__(self, **kwargs): # The current graph parsing requires us to update all outputs of # a module that "accepted" the input. # Update means changing the original producer_args for the bound port. - producer = self._bound_input_module[port_name] - for _, output_tensor in self._bound_output_tensors.items(): + producer = self._bound_input_modules[port_name] + for _, output_tensor in self._bound_output_tensors_default.items(): if output_tensor.producer == producer: # Set "input port value" to new content - which indicates tensor (and producer) # that will be used during graph backward traverse. @@ -149,7 +153,7 @@ def __call__(self, **kwargs): # Get the name of the ouput port. out_name = list(output_port_defs)[0] # Simply pass the bound tensor. - results = self._bound_output_tensors[out_name] + results = self._bound_output_tensors_default[out_name] # BUT UPDATE THE inputs to it!! # Bind the output ports. @@ -157,7 +161,7 @@ def __call__(self, **kwargs): else: result = [] - for _, tensor in self._bound_output_tensors.items(): + for _, tensor in self._bound_output_tensors_default.items(): result.append(tensor) # Creating ad-hoc class for returning from module's forward pass. @@ -190,27 +194,38 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: Returns: A (dict) of module's output ports names to NeuralTypes mapping """ - return self._bound_output_ports + #print("getter!") + return self._bound_output_ports_default + + #@output_ports.setter + #def output_ports(self, ports): + # print("setter!") + # self._bound_output_ports_default = ports def __enter__(self): """ Activates given graph as current. """ - # print("Entering graph: ", self._name) + # print("Entering graph: ", self.name) self._app_state.active_graph = self return self def __exit__(self, exc_type, exc_value, exc_traceback): """ Deactivates current graph. """ - # print("Exiting graph: ", self._name) + # print("Exiting graph: ", self.name) self._app_state.active_graph = None # if exc_type: # print(f'exc_type: {exc_type}') # print(f'exc_value: {exc_value}') # print(f'exc_traceback: {exc_traceback}') + @property + def name(self): + """ Returns graph name. """ + return self._name + def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) - desc = "`{}` ({}):\n".format(self._name, len(self._steps)) + desc = "`{}` ({}):\n".format(self.name, len(self._steps)) for op in self._steps: desc = desc + " {}\n".format(type(op[0]).__name__) return desc @@ -229,7 +244,7 @@ def __len__(self): return len(self._modules) def list_modules(self): - desc = "{} ({}):\n".format(retrieve_variable_name(self), len(self)) + desc = "{} ({}):\n".format(self.name, len(self)) for key, value in self._modules.items(): desc += " * `{}` ({})\n".format(key, value ) return desc @@ -238,15 +253,11 @@ def record_step(self, module, inputs): """ Records the operation (module plus passed inputs) on a list. """ - # Get module name. - module_name = retrieve_variable_name(module) - #print("module_name: ", module_name) - # Check if module with that name already exists. - if module_name in self._modules.keys(): - raise KeyError("Neural Graph already contains a module named {}".format(module_name)) + if module.name in self._modules.keys(): + raise KeyError("Neural Graph already contains a module named {}".format(module.name)) # Add module to list of modules. - self._modules[module_name] = module + self._modules[module.name] = module # Add step. self._steps.append([module, inputs]) @@ -259,7 +270,7 @@ def bind_input(self, port_name, port_definition, bound_module): # Indicate that this tensor is missing and has to be provided! self._bound_input_tensors[port_name] = None # Additionally, remember the bound module - self._bound_input_module[port_name] = bound_module + self._bound_input_modules[port_name] = bound_module def bind_outputs(self, output_port_defs, output_values): # print("Binding ALL outputs: defs = `{}`, values = `{}`".format(output_port_defs, output_values)) @@ -271,10 +282,10 @@ def bind_outputs(self, output_port_defs, output_values): # ) # ) # Copy the definition of the port to graph input port definition. - self._bound_output_ports[output_name] = output_definition + self._bound_output_ports_default[output_name] = output_definition # Bind output tensors. - self._bound_output_tensors[output_name] = output_value + self._bound_output_tensors_default[output_name] = output_value # Additionally, store all output tensors. # self._all_output_tensors[output_name] = output_value @@ -288,10 +299,10 @@ def show_bound_inputs(self): print(" * `{}`: `{}` ({})".format(key, value, type(value))) def show_bound_outputs(self): - print("bound output ports: ") - for key, value in self._bound_output_ports.items(): + print("bound (default) output ports: ") + for key, value in self._bound_output_ports_default.items(): print(" * `{}`: `{}` ({})".format(key, value, type(value))) - print("bound output tensors: ") - for key, value in self._bound_output_tensors.items(): + print("bound (default) output tensors: ") + for key, value in self._bound_output_tensors_default.items(): print(" * `{}`: `{}` ({})".format(key, value, type(value))) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 3f4fb124c9ac..a94e70b0b737 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -63,10 +63,13 @@ class NeuralModule(NeuralInterface): """Abstract class that every Neural Module must inherit from. """ - def __init__(self): + def __init__(self, name=None): # Call integrace constructor. super().__init__() + # Save name. + self._name = name + # Get default factory. self._factory = NeuralModuleFactory.get_default_factory() @@ -83,13 +86,6 @@ def __init__(self): # Retrieve dictionary of parameters (keys, values) passed to init. self._init_params = self.__extract_init_params() - # Pint the types of the values. - # for key, value in self._init_params.items(): - # print("{}: {} ({})".format(key, value, type(value))) - - # Validate the parameters. - # self._validate_params(self._init_params) - @property def init_params(self) -> Optional[Dict]: """ @@ -369,6 +365,11 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) ) return obj + @property + def name(self): + """ Returns graph name. """ + return self._name + @deprecated(version=0.11) @staticmethod def create_ports(**kwargs): @@ -414,7 +415,7 @@ def _disabled_deployment_output_ports(self) -> Optional[Set[str]]: """ return set([]) - def _prepare_for_deployment(self) -> None: + def _prepare_for_deployment(self) -> None: """Patch the module if required to prepare for deployment """ diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 30707e573df5..255316fef491 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -24,5 +24,3 @@ from .argparse import NemoArgParser from .exp_logging import ExpManager, get_logger from .helpers import * - -from .retrieve_variable_name import retrieve_variable_name \ No newline at end of file diff --git a/nemo/utils/retrieve_variable_name.py b/nemo/utils/retrieve_variable_name.py deleted file mode 100644 index f59d1eca40bd..000000000000 --- a/nemo/utils/retrieve_variable_name.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import inspect - -def retrieve_variable_name(var): - """ - Function returns the name of the variable. - Throws the KeyError exception is name was not found. - - Args: - var: Variable. - - Returns: - String representing the variable name. - """ - for fi in reversed(inspect.stack()): - names = [var_name for var_name, var_val in fi.frame.f_locals.items() if var_val is var] - if len(names) > 0: - return names[0] - else: - raise KeyError("Cannot retrieve the name of object {} as it is not present on the stack (yet)".format(var)) From 5221fef7584f6c50b9da615b85d9e97f33865615 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 7 Apr 2020 13:45:08 -0700 Subject: [PATCH 009/106] implemented NameRegistry basic functionality, test5 working, unit tests failing Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests5.py | 10 ++- nemo/backends/pytorch/nm.py | 1 - nemo/backends/pytorch/tutorials/toys.py | 1 - nemo/core/neural_graph_manager.py | 47 ++++++----- nemo/core/neural_modules.py | 84 ++++++++++++++++--- 5 files changed, 108 insertions(+), 35 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py index 52c4c1a78982..c6a79eaafebd 100644 --- a/examples/start_here/graph_composition_integration_tests5.py +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -26,10 +26,12 @@ nf = nemo.core.NeuralModuleFactory() # Instantiate the necessary neural modules. -dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128, name="dl_training") +dl_training = nemo.tutorials.RealFunctionDataLayer(n=1000, batch_size=32, name="dl_training") fx = nemo.tutorials.TaylorNet(dim=4, name="fx") -print(fx.name) +fx2 = nemo.tutorials.TaylorNet(dim=4) +fx3 = nemo.tutorials.TaylorNet(dim=4) loss = nemo.tutorials.MSELoss(name="loss") +loss2 = nemo.tutorials.MSELoss() logging.info( "This example shows how one can access modules nested in a graph." @@ -39,6 +41,8 @@ with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: # Bind the input. _ = fx(x=trainable_module) + _ = fx2(x=trainable_module) + _ = fx3(x=trainable_module) # All outputs will be bound by default. # Compose two graphs into final graph. @@ -49,6 +53,8 @@ p = trainable_module(x=x) # Pass both of them to loss. lss = loss(predictions=p, target=t) + lss2 = loss2(predictions=p, target=t) + print(trainable_module.list_modules()) diff --git a/nemo/backends/pytorch/nm.py b/nemo/backends/pytorch/nm.py index 06938a66ba79..734ab9d60b50 100644 --- a/nemo/backends/pytorch/nm.py +++ b/nemo/backends/pytorch/nm.py @@ -35,7 +35,6 @@ def __init__(self): """ def __init__(self, pretrained_model_name=None, name=None): - print("TrainableNM: ", name) NeuralModule.__init__(self, name) # For NeuralModule API nn.Module.__init__(self) # For PyTorch API diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index b524e2c28a26..7880d3f1a0e2 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -43,7 +43,6 @@ def __init__(self, dim, name=None): dim: Number of dimensions (number of terms in Taylor series). name: Name of the module instance """ - print("TaylorNet: ", name) super().__init__(name=name) # And of Neural Modules specific part. Rest is Pytorch code diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph_manager.py index ddf9f4e0a08f..d644d0470184 100644 --- a/nemo/core/neural_graph_manager.py +++ b/nemo/core/neural_graph_manager.py @@ -25,11 +25,11 @@ class NeuralGraphManager(object): def __init__(self): """ - Constructor. Initializes the manager. + Constructor. Initializes the manager. - Args: - operation_mode: Graph operation mode, that will be propagated along modules during graph creation. - [training | eval] + Args: + operation_mode: Graph operation mode, that will be propagated along modules during graph creation. + [training | evaluation | both] """ self._active_graph = None self._graphs = {} @@ -37,16 +37,17 @@ def __init__(self): def register_graph(self, graph): """ Registers a new graph. """ # Create a unigue name. - # Add it to the list. unique_name = self.__generate_unique_graph_name(graph._name) + # Add graph to the list. self._graphs[unique_name] = graph @property def graphs(self): - """ Property returns the list of graphs. + """ + Property returns the list of graphs. - Returns: - List of created graphs. + Returns: + List of created graphs. """ return self._graphs @@ -61,15 +62,15 @@ def summary(self): @property def active_graph(self): """ - Property returns the active graph. If there is no active graph, creates a new one. + Property returns the active graph. If there is no active graph, creates a new one. - Returns: - Active graph + Returns: + Active graph """ # Create a new graph - training is the default. if self._active_graph is None: - # Store graph on the list. - new_graph = NeuralGraph(operation_mode=OperationMode.training) + # Create a new "default" graph. Default mode: both. + new_graph = NeuralGraph(operation_mode=OperationMode.both) self.register_graph(new_graph) # Set the newly created graph as active. self._active_graph = new_graph @@ -79,10 +80,11 @@ def active_graph(self): @active_graph.setter def active_graph(self, graph): - """ Property sets the active graph. + """ + Property sets the active graph. - Args: - graph: Neural graph object that will become active. + Args: + graph: Neural graph object that will become active. """ # Activate the graph. self._active_graph = graph @@ -103,9 +105,10 @@ def __generate_unique_graph_name(self, name): def __len__(self): """ + Returns number of existing graphs. - :return: Number of created neural graphs. - + Returns: + Number of created neural graphs. """ return len(self._graphs) @@ -113,9 +116,11 @@ def __getitem__(self, key): """ Value getter function. - :param key: Graph name. + Args: + key: Graph name. - :return: Associated graph. + Returns: + Associated graph. """ - # Retrieve the value. + # Retrieve the graph. return self._graphs[key] diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index a94e70b0b737..76e4e02e4e99 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -59,16 +59,85 @@ class WeightShareTransform(Enum): ) -class NeuralModule(NeuralInterface): - """Abstract class that every Neural Module must inherit from. +import weakref + +class NameRegistry(object): + """ + Class used for storing a list of object instances, generating unique names and monitoring their `uniqueness`. """ + _instances = set() def __init__(self, name=None): - # Call integrace constructor. + """ + Sets object name and registers it on a list. + """ super().__init__() - # Save name. - self._name = name + # Set object name. + if name is None: + self._name = self.__generate_unique_name(type(self).__name__) + else: + # Check if name is unique. + for ref in self._instances: + if ref().name == name: + NameError("Module with name `{}` already exists!".format(name)) + # Ok, set the object's name. + self._name = name + + # Register object. + self._instances.add(weakref.ref(self)) + + @property + def name(self): + """ Returns the object name. """ + return self._name + + def __generate_unique_name(self, base_name): + """ + Generates a new unique name by adding postfix (number) to base name. + + Args: + base_name: Base name. + Returns: + Generated name, + """ + # Iterate through numbers. + postfix = 0 + name_unique = False + while not name_unique: + # Generate name. + new_name = base_name + str(postfix) + name_unique = True + for ref in self._instances: + if ref().name == new_name: + # Sadly name not unique. + name_unique = False + break + # Increment index. + postfix += 1 + return new_name + + @classmethod + def get_instances(cls): + dead = set() + for ref in cls._instances: + obj = ref() + if obj is not None: + yield obj + else: + dead.add(ref) + cls._instances -= dead + +class NeuralModule(NeuralInterface, NameRegistry): + """ + Abstract class that every Neural Module must inherit from. + """ + + def __init__(self, name=None): + # Initialize inferface. + NeuralInterface.__init__(self) + # Auto-register name. + NameRegistry.__init__(self, name=name) # Get default factory. self._factory = NeuralModuleFactory.get_default_factory() @@ -365,11 +434,6 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) ) return obj - @property - def name(self): - """ Returns graph name. """ - return self._name - @deprecated(version=0.11) @staticmethod def create_ports(**kwargs): From dd9af75e9b167c6fc004d8be07cf43330e86bf4a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 15:30:36 -0700 Subject: [PATCH 010/106] example 5 working, ObjectRegistry finished, app_state moved to utils Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests5.py | 1 + nemo/core/__init__.py | 1 - nemo/core/neural_graph.py | 9 +- nemo/core/neural_interface.py | 21 +++- nemo/core/neural_modules.py | 88 ++------------ nemo/utils/__init__.py | 7 +- nemo/{core => utils}/app_state.py | 18 ++- nemo/utils/object_registry.py | 108 ++++++++++++++++++ tests/unit/{core => utils}/test_app_state.py | 2 +- 9 files changed, 158 insertions(+), 97 deletions(-) rename nemo/{core => utils}/app_state.py (84%) create mode 100644 nemo/utils/object_registry.py rename tests/unit/{core => utils}/test_app_state.py (97%) diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py index c6a79eaafebd..f06cf098610c 100644 --- a/examples/start_here/graph_composition_integration_tests5.py +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -30,6 +30,7 @@ fx = nemo.tutorials.TaylorNet(dim=4, name="fx") fx2 = nemo.tutorials.TaylorNet(dim=4) fx3 = nemo.tutorials.TaylorNet(dim=4) +print(fx3.name) loss = nemo.tutorials.MSELoss(name="loss") loss2 = nemo.tutorials.MSELoss() diff --git a/nemo/core/__init__.py b/nemo/core/__init__.py index 7979d149e549..56882194c05b 100644 --- a/nemo/core/__init__.py +++ b/nemo/core/__init__.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo.core.app_state import AppState from nemo.core.callbacks import * from nemo.core.neural_factory import * from nemo.core.neural_graph import NeuralGraph diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 46735538e03e..7315e11cb754 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -44,8 +44,8 @@ def __init__(self, operation_mode, name=None): [training | eval] name: Name of the graph (optional) """ - # Call integrace constructor. - super().__init__() + # Initialize the inferface. + super().__init__(name) # Store name and operation mode. self._operation_mode = operation_mode @@ -217,11 +217,6 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # print(f'exc_value: {exc_value}') # print(f'exc_traceback: {exc_traceback}') - @property - def name(self): - """ Returns graph name. """ - return self._name - def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py index b646491ba723..ec410c669ed9 100644 --- a/nemo/core/neural_interface.py +++ b/nemo/core/neural_interface.py @@ -33,15 +33,18 @@ class NeuralInterface(ABC): graph, e.g. get_weights, tie_weights, ) """ - def __init__(self): - """ Constructor. Set application state. """ - # Get access to app state. - self._app_state = nemo.core.app_state.AppState() + def __init__(self, name): + """ Constructor. Sets the application state. """ + # Copy the name. As names should be unique in module/graph scope, this should be handled additionally + # in their constructors. + self._name = name + # Create access to the app state. + self._app_state = nemo.utils.app_state.AppState() @property @abstractmethod def input_ports(self) -> Optional[Dict[str, NeuralType]]: - """Returns definitions of module input ports + """ Returns definitions of module input ports Returns: A (dict) of module's input ports names to NeuralTypes mapping @@ -50,7 +53,7 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: @property @abstractmethod def output_ports(self) -> Optional[Dict[str, NeuralType]]: - """Returns definitions of module output ports + """ Returns definitions of module output ports Returns: A (dict) of module's output ports names to NeuralTypes mapping @@ -65,3 +68,9 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ + + @property + def name(self): + """ Returns the object name. """ + return self._name + diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 76e4e02e4e99..6eb5013ee8b3 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -58,86 +58,23 @@ class WeightShareTransform(Enum): "PretrainedModleInfo", ("pretrained_model_name", "description", "parameters", "location"), ) - -import weakref - -class NameRegistry(object): +class NeuralModule(NeuralInterface): """ - Class used for storing a list of object instances, generating unique names and monitoring their `uniqueness`. + Abstract class that every Neural Module must inherit from. """ - _instances = set() def __init__(self, name=None): - """ - Sets object name and registers it on a list. - """ - super().__init__() - - # Set object name. - if name is None: - self._name = self.__generate_unique_name(type(self).__name__) - else: - # Check if name is unique. - for ref in self._instances: - if ref().name == name: - NameError("Module with name `{}` already exists!".format(name)) - # Ok, set the object's name. - self._name = name - - # Register object. - self._instances.add(weakref.ref(self)) - - @property - def name(self): - """ Returns the object name. """ - return self._name - - def __generate_unique_name(self, base_name): - """ - Generates a new unique name by adding postfix (number) to base name. + # Initialize the inferface. + super().__init__(name) - Args: - base_name: Base name. - Returns: - Generated name, - """ - # Iterate through numbers. - postfix = 0 - name_unique = False - while not name_unique: - # Generate name. - new_name = base_name + str(postfix) - name_unique = True - for ref in self._instances: - if ref().name == new_name: - # Sadly name not unique. - name_unique = False - break - # Increment index. - postfix += 1 - return new_name - - @classmethod - def get_instances(cls): - dead = set() - for ref in cls._instances: - obj = ref() - if obj is not None: - yield obj - else: - dead.add(ref) - cls._instances -= dead + # Retrieve dictionary of parameters (keys, values) passed to init. + self._init_params = self.__extract_init_params() -class NeuralModule(NeuralInterface, NameRegistry): - """ - Abstract class that every Neural Module must inherit from. - """ + # Get object UUID. + self._uuid = str(uuid.uuid4()) - def __init__(self, name=None): - # Initialize inferface. - NeuralInterface.__init__(self) - # Auto-register name. - NameRegistry.__init__(self, name=name) + # Register module and store the generated name. + self._name = self._app_state.register_module(self, name) # Get default factory. self._factory = NeuralModuleFactory.get_default_factory() @@ -149,11 +86,7 @@ def __init__(self, name=None): # Optimization level. self._opt_level = self._factory.optim_level - # Get object UUID. - self._uuid = str(uuid.uuid4()) - # Retrieve dictionary of parameters (keys, values) passed to init. - self._init_params = self.__extract_init_params() @property def init_params(self) -> Optional[Dict]: @@ -196,6 +129,7 @@ def __extract_init_params(self): # Return parameters. return init_params + def __validate_params(self, params): """ Checks whether dictionary contains parameters being primitive types (string, int, float etc.) diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 255316fef491..d67c8ee31b77 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -1,7 +1,6 @@ -# ! /usr/bin/python # -*- coding: utf-8 -*- - -# Copyright 2020 NVIDIA. All Rights Reserved. +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,3 +23,5 @@ from .argparse import NemoArgParser from .exp_logging import ExpManager, get_logger from .helpers import * +from nemo.utils.app_state import AppState +from nemo.utils.object_registry import ObjectRegistry diff --git a/nemo/core/app_state.py b/nemo/utils/app_state.py similarity index 84% rename from nemo/core/app_state.py rename to nemo/utils/app_state.py index 52d9bd565280..19e55eb88d26 100644 --- a/nemo/core/app_state.py +++ b/nemo/utils/app_state.py @@ -1,6 +1,4 @@ -# ! /usr/bin/python # -*- coding: utf-8 -*- - # ============================================================================= # Copyright (c) 2020 NVIDIA. All Rights Reserved. # @@ -63,6 +61,8 @@ def __init__(self, device=None): self._device = nemo.core.DeviceType.GPU else: self._device = device + # Create registers. + self._module_registry = nemo.utils.ObjectRegistry("module") self._neural_graph_manager = nemo.core.NeuralGraphManager() @property @@ -78,6 +78,20 @@ def register_graph(self, graph): """ Registers a new graph. """ self._neural_graph_manager.register_graph(graph) + def register_module(self, module, name): + """ + Registers a module using the provided name. + If name is none - generates new unique name. + + Args: + module: A Neural Module object to be registered. + name: A "proposition" of module name. + + Returns: + A unique name (proposition or newly generated name). + """ + return self._module_registry.register(module, name) + @property def active_graph(self): """ Property returns the active graph. diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py new file mode 100644 index 000000000000..4a7d96922d8a --- /dev/null +++ b/nemo/utils/object_registry.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from weakref import WeakSet + +class ObjectRegistry(WeakSet): + """ + Registry used for storing references to objects, generating unique names and monitoring their `uniqueness`. + """ + + def __init__(self, base_type_name): + """ + Stores base type name. + """ + super().__init__() + self._base_type_name = base_type_name + + def register(self, new_obj, name): + """ + Registers a new object using the provided name. + If name is none - generates new unique name. + + Args: + new_obj: An object to be registered. + name: A "proposition" of object name. + + Returns: + A unique name (proposition or newly generated name). + """ + + # Check if object is already in a set. + if new_obj in self: + # Return its name. + return new_obj.name + + # Check object name. + if name is None: + # Generate a new, unique name. + unique_name = self.__generate_unique_name() + else: + # Check if name is unique. + for obj in self: + if obj.name == name: + raise NameError("A {} with name `{}` already exists!".format(self._base_type_name, name)) + # Ok, it is unique. + unique_name = name + + # Finally, add object to the set. + self.add(new_obj) + + # Return the name. + return unique_name + + def __generate_unique_name(self): + """ + Generates a new unique name by adding postfix (number) to base name. + + Returns: + A generated unique name. + """ + # Iterate through numbers. + postfix = 0 + name_unique = False + while not name_unique: + # Generate name. + new_name = self._base_type_name + str(postfix) + name_unique = True + # Check uniqueneess. + for obj in self: + if obj.name == new_name: + # Sadly name is not unique. + name_unique = False + break + # Increment index. + postfix += 1 + return new_name + + def __getitem__(self, key): + """ + Object getter function. + + Args: + key: Object name. + + Returns: + Associated . + """ + # Search for an object with a given name. + for obj in self: + # Retrieve object + if obj.name == key: + return obj + # Else: seems that there is no object with that name. + raise KeyError("A {} with name `{}` don't exists!".format(self._base_type_name, key)) diff --git a/tests/unit/core/test_app_state.py b/tests/unit/utils/test_app_state.py similarity index 97% rename from tests/unit/core/test_app_state.py rename to tests/unit/utils/test_app_state.py index 71361003810a..cd2f846c3ba4 100644 --- a/tests/unit/core/test_app_state.py +++ b/tests/unit/utils/test_app_state.py @@ -27,7 +27,7 @@ class TestAppState(TestCase): @pytest.mark.unit - def test_shared_graph(self): + def test_value_sharing(self): # Create first instance of AppState. x = AppState() x.test_value = "ala" From ab3562121b59b1dff3bece944f640f6ba36408ab Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 15:34:03 -0700 Subject: [PATCH 011/106] test app_state fixed Signed-off-by: Tomasz Kornuta --- tests/unit/utils/test_app_state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/unit/utils/test_app_state.py b/tests/unit/utils/test_app_state.py index cd2f846c3ba4..a5a1e140a2cd 100644 --- a/tests/unit/utils/test_app_state.py +++ b/tests/unit/utils/test_app_state.py @@ -17,15 +17,13 @@ # limitations under the License. # ============================================================================= -from unittest import TestCase - import pytest from ruamel.yaml import YAML -from nemo.core import AppState +from nemo.utils.app_state import AppState -class TestAppState(TestCase): +class TestAppState(): @pytest.mark.unit def test_value_sharing(self): # Create first instance of AppState. @@ -33,7 +31,8 @@ def test_value_sharing(self): x.test_value = "ala" # Create second instance of AppState and test value. y = AppState() - self.assertEqual(y.test_value, "ala") + assert y.test_value == "ala" + # Change second instance and test first one. y.test_value = "ola" - self.assertEqual(x.test_value, "ola") + assert x.test_value == "ola" From bbd35893b6656a82f60221cf6c6436ad08d68a1f Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 15:38:52 -0700 Subject: [PATCH 012/106] TokenClassification self.name fix Signed-off-by: Tomasz Kornuta --- .../nlp/nm/trainables/common/token_classification_nm.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index 93aee4e9bf8d..6fcd080d9a63 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -131,9 +131,8 @@ def __init__( dropout=0.0, use_transformer_pretrained=True, ): - super().__init__() + super().__init__(name) - self.name = name self.mlp = MultiLayerPerceptron(hidden_size, num_classes, self._device, num_layers, activation, log_softmax) self.dropout = nn.Dropout(dropout) if use_transformer_pretrained: @@ -141,11 +140,7 @@ def __init__( # self.to(self._device) # sometimes this is necessary def __str__(self): - name = TrainableNM.__str__(self) - - if self.name: - name = self.name + name - return name + return self.name def forward(self, hidden_states): hidden_states = self.dropout(hidden_states) From 56efff5ef1353d1c509bcf5f58b313d9eb11cf15 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 15:51:21 -0700 Subject: [PATCH 013/106] ObjectRegistry tests Signed-off-by: Tomasz Kornuta --- .../common/token_classification_nm.py | 1 + tests/unit/utils/test_app_state.py | 3 - tests/unit/utils/test_object_registry.py | 65 +++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/unit/utils/test_object_registry.py diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index 6fcd080d9a63..0561321781f7 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -131,6 +131,7 @@ def __init__( dropout=0.0, use_transformer_pretrained=True, ): + # Pass name up the module class hierarchy. super().__init__(name) self.mlp = MultiLayerPerceptron(hidden_size, num_classes, self._device, num_layers, activation, log_softmax) diff --git a/tests/unit/utils/test_app_state.py b/tests/unit/utils/test_app_state.py index a5a1e140a2cd..5fcae6ee6d35 100644 --- a/tests/unit/utils/test_app_state.py +++ b/tests/unit/utils/test_app_state.py @@ -1,6 +1,5 @@ # ! /usr/bin/python # -*- coding: utf-8 -*- - # ============================================================================= # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # @@ -18,11 +17,9 @@ # ============================================================================= import pytest -from ruamel.yaml import YAML from nemo.utils.app_state import AppState - class TestAppState(): @pytest.mark.unit def test_value_sharing(self): diff --git a/tests/unit/utils/test_object_registry.py b/tests/unit/utils/test_object_registry.py new file mode 100644 index 000000000000..809e953e669c --- /dev/null +++ b/tests/unit/utils/test_object_registry.py @@ -0,0 +1,65 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest + +from nemo.utils.object_registry import ObjectRegistry + +class TestAppState(): + + @pytest.mark.unit + def test_registry(self): + """ Tests registry reference management. """ + # Crete new registry. + registry = ObjectRegistry("object") + + class MockupObjectClass: + def __init__(self, name=None): + # Store name generated by the registry. + self.name = registry.register(self, name) + + # Test object uniqueness. + c1 = MockupObjectClass("c1") + c1_ref = registry["c1"] + assert c1_ref == c1 + + # Test name uniqueness. + c2 = MockupObjectClass("c2") + with pytest.raises(NameError): + _ = MockupObjectClass("c2") + + # Test unique names generation. + c3 = MockupObjectClass() + c4 = MockupObjectClass() + assert c4.name == "object1" + + # Check objects. + assert len(registry) == 4 + + # Delete all objects - aside of reference! + del(c1) + del(c2) + del(c3) + del(c4) + assert len(registry) == 1 + with pytest.raises(KeyError): + registry["c4"] + + # Delete the last object. + del(c1_ref) + assert len(registry) == 0 From c838f1356683f576549a3493c7edc9c5662bf5e9 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 16:18:18 -0700 Subject: [PATCH 014/106] cleanup of neural graphs class, NG manager inheriting from ObjectRegistry, starting to work on graph unit tests Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests1_2.py | 6 +- .../graph_composition_integration_tests1_3.py | 3 +- nemo/core/neural_graph.py | 42 ++++++---- nemo/core/neural_graph_manager.py | 83 +++---------------- nemo/utils/app_state.py | 33 ++++++-- 5 files changed, 69 insertions(+), 98 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests1_2.py b/examples/start_here/graph_composition_integration_tests1_2.py index cfce01d5f62c..1df2408d8ecd 100644 --- a/examples/start_here/graph_composition_integration_tests1_2.py +++ b/examples/start_here/graph_composition_integration_tests1_2.py @@ -18,7 +18,7 @@ # ============================================================================= import nemo -from nemo.core import AppState, NeuralGraph, OperationMode +from nemo.core import NeuralGraph, OperationMode logging = nemo.logging @@ -36,7 +36,7 @@ # Create the g0 graph. g0 = NeuralGraph(operation_mode=OperationMode.training) -# Activate the "g0 graph context" "manually" - all operations will be recorded to g0. +# Activate the "g0 graph context" "manually" - all steps will be recorded to g0. g0.activate() # Define g0 - connections between the modules. @@ -44,7 +44,7 @@ p = m2(x=x) lss = loss(predictions=p, target=t) -# Deactivate the "g0 graph context" (this is really optional, as long as there are no other operations). +# Deactivate the "g0 graph context" (this is really optional, as long as there are no other steps to be recorded). g0.deactivate() # SimpleLossLoggerCallback will print loss values to console. diff --git a/examples/start_here/graph_composition_integration_tests1_3.py b/examples/start_here/graph_composition_integration_tests1_3.py index 2fbd6d7c9e08..21e865b2de6d 100755 --- a/examples/start_here/graph_composition_integration_tests1_3.py +++ b/examples/start_here/graph_composition_integration_tests1_3.py @@ -18,7 +18,8 @@ # ============================================================================= import nemo -from nemo.core import AppState, NeuralGraph, OperationMode +from nemo.core import NeuralGraph, OperationMode +from nemo.utils.app_state import AppState logging = nemo.logging diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 7315e11cb754..3d0a89389f4a 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -33,8 +33,6 @@ class NeuralGraph(NeuralInterface): Neural Graph class stores dynamically defined graphs of connected Neural Modules. """ - - def __init__(self, operation_mode, name=None): """ Constructor. Initializes graph variables. @@ -47,13 +45,11 @@ def __init__(self, operation_mode, name=None): # Initialize the inferface. super().__init__(name) + # Register graph. + self._name = self._app_state.register_graph(self, name) + # Store name and operation mode. self._operation_mode = operation_mode - if name is None: - # Simply take the name of operation. - self._name = str(self._operation_mode)[14:] - else: - self._name = name # Input ports and tensors - empty for now. self._bound_input_ports = {} @@ -74,8 +70,6 @@ def __init__(self, operation_mode, name=None): # "Steps": ordered execution of modules in a graph. self._steps = [] - # Register graph. - self._app_state.register_graph(self) def __call__(self, **kwargs): """ @@ -203,19 +197,33 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: # self._bound_output_ports_default = ports def __enter__(self): - """ Activates given graph as current. """ - # print("Entering graph: ", self.name) + """ + Activates this graph. + + Returns: + The graph object. + """ self._app_state.active_graph = self return self def __exit__(self, exc_type, exc_value, exc_traceback): - """ Deactivates current graph. """ - # print("Exiting graph: ", self.name) + """ + Deactivates the current graph. + """ self._app_state.active_graph = None - # if exc_type: - # print(f'exc_type: {exc_type}') - # print(f'exc_value: {exc_value}') - # print(f'exc_traceback: {exc_traceback}') + + def activate(self): + """ + Activates this graph. + """ + self._app_state.active_graph = self + + def deactivate(self): + """ + Deactivates the current graph. + """ + self._app_state.active_graph = None + def __str__(self): """ Prints a nice summary. """ diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph_manager.py index d644d0470184..1095f9a569a9 100644 --- a/nemo/core/neural_graph_manager.py +++ b/nemo/core/neural_graph_manager.py @@ -16,62 +16,41 @@ # limitations under the License. # ============================================================================= -# from collections.abc import Mapping +from nemo.utils.object_registry import ObjectRegistry from nemo.core.neural_factory import OperationMode from nemo.core.neural_graph import NeuralGraph -class NeuralGraphManager(object): +class NeuralGraphManager(ObjectRegistry): def __init__(self): """ - Constructor. Initializes the manager. - - Args: - operation_mode: Graph operation mode, that will be propagated along modules during graph creation. - [training | evaluation | both] + Constructor. Initializes the manager. Sets active graph to None. """ + super().__init__("graph") self._active_graph = None - self._graphs = {} - - def register_graph(self, graph): - """ Registers a new graph. """ - # Create a unigue name. - unique_name = self.__generate_unique_graph_name(graph._name) - # Add graph to the list. - self._graphs[unique_name] = graph - - @property - def graphs(self): - """ - Property returns the list of graphs. - - Returns: - List of created graphs. - """ - return self._graphs def summary(self): """ Prints a nice summary. """ # TODO: a nicer summary. ;) desc = "" - for name, graph in self._graphs.items(): - desc = desc + "`{}`: {}\n".format(name, graph) + for graph in self: + desc = desc + "`{}`: {}\n".format(graph.name, graph) return desc @property def active_graph(self): """ - Property returns the active graph. If there is no active graph, creates a new one. + Property returns the active graph. If there is no active graph, creates a new one. - Returns: - Active graph + Returns: + The active graph object. """ # Create a new graph - training is the default. if self._active_graph is None: # Create a new "default" graph. Default mode: both. new_graph = NeuralGraph(operation_mode=OperationMode.both) - self.register_graph(new_graph) + new_graph.name = self.register(new_graph, None) # Set the newly created graph as active. self._active_graph = new_graph @@ -81,46 +60,10 @@ def active_graph(self): @active_graph.setter def active_graph(self, graph): """ - Property sets the active graph. + Property sets the active graph. - Args: - graph: Neural graph object that will become active. + Args: + graph: Neural graph object that will become active. """ # Activate the graph. self._active_graph = graph - - def __generate_unique_graph_name(self, name): - """ Generates a new unique name by adding postfix (number). """ - # Simply return the same name as long as it is unique. - if name not in self._graphs.keys(): - return name - - # Iterate through numbers. - postfix = 1 - new_name = name + str(postfix) - while new_name in self._graphs.keys(): - postfix = postfix + 1 - new_name = name + str(postfix) - return new_name - - def __len__(self): - """ - Returns number of existing graphs. - - Returns: - Number of created neural graphs. - """ - return len(self._graphs) - - def __getitem__(self, key): - """ - Value getter function. - - Args: - key: Graph name. - - Returns: - Associated graph. - """ - # Retrieve the graph. - return self._graphs[key] diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index 19e55eb88d26..74ad259d9215 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -65,23 +65,28 @@ def __init__(self, device=None): self._module_registry = nemo.utils.ObjectRegistry("module") self._neural_graph_manager = nemo.core.NeuralGraphManager() + @property + def modules(self): + """ Property returning the existing modules. + + Returns: + Existing modules (a set object). + """ + return self._module_registry + @property def graphs(self): - """ Property returns the graph manager. + """ Property returning the existing graphs. Returns: - List of created graphs + Existing graphs (a set object). """ return self._neural_graph_manager - def register_graph(self, graph): - """ Registers a new graph. """ - self._neural_graph_manager.register_graph(graph) - def register_module(self, module, name): """ Registers a module using the provided name. - If name is none - generates new unique name. + If name is none - generates a new unique name. Args: module: A Neural Module object to be registered. @@ -92,6 +97,20 @@ def register_module(self, module, name): """ return self._module_registry.register(module, name) + def register_graph(self, graph, name): + """ + Registers a new graph using the provided name. + If name is none - generates a new unique name. + + Args: + graph: A Neural Graph object to be registered. + name: A "proposition" of graph name. + + Returns: + A unique name (proposition or newly generated name). + """ + return self._neural_graph_manager.register(graph, name) + @property def active_graph(self): """ Property returns the active graph. From 115300172104e139f95e2ea406790e6a3adeda12 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 16:31:01 -0700 Subject: [PATCH 015/106] fixes to graph and manager, NeMo tests passing (locally) Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 4 ++-- nemo/core/neural_graph_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 3d0a89389f4a..522cfad0fcba 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -257,8 +257,8 @@ def record_step(self, module, inputs): Records the operation (module plus passed inputs) on a list. """ # Check if module with that name already exists. - if module.name in self._modules.keys(): - raise KeyError("Neural Graph already contains a module named {}".format(module.name)) + #if module.name in self._modules.keys(): + # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) # Add module to list of modules. self._modules[module.name] = module diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph_manager.py index 1095f9a569a9..2b3662cf94f4 100644 --- a/nemo/core/neural_graph_manager.py +++ b/nemo/core/neural_graph_manager.py @@ -50,7 +50,7 @@ def active_graph(self): if self._active_graph is None: # Create a new "default" graph. Default mode: both. new_graph = NeuralGraph(operation_mode=OperationMode.both) - new_graph.name = self.register(new_graph, None) + new_graph._name = self.register(new_graph, None) # Set the newly created graph as active. self._active_graph = new_graph From 9df6568fe0b0414f666c16830fa9233ef9eb2552 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 16:33:07 -0700 Subject: [PATCH 016/106] code formatting Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests5.py | 8 +++----- nemo/backends/pytorch/tutorials/toys.py | 2 +- nemo/core/neural_graph.py | 13 ++++++------- nemo/core/neural_graph_manager.py | 3 +-- nemo/core/neural_interface.py | 3 +-- nemo/core/neural_modules.py | 6 ++---- nemo/utils/object_registry.py | 3 ++- tests/unit/utils/test_app_state.py | 3 ++- tests/unit/utils/test_object_registry.py | 12 ++++++------ 9 files changed, 24 insertions(+), 29 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py index f06cf098610c..6fec1be463f3 100644 --- a/examples/start_here/graph_composition_integration_tests5.py +++ b/examples/start_here/graph_composition_integration_tests5.py @@ -34,9 +34,7 @@ loss = nemo.tutorials.MSELoss(name="loss") loss2 = nemo.tutorials.MSELoss() -logging.info( - "This example shows how one can access modules nested in a graph." -) +logging.info("This example shows how one can access modules nested in a graph.") # Build the training graph. with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: @@ -55,7 +53,7 @@ # Pass both of them to loss. lss = loss(predictions=p, target=t) lss2 = loss2(predictions=p, target=t) - + print(trainable_module.list_modules()) @@ -71,4 +69,4 @@ _ = training_graph["other_module"] except KeyError as e: print("Got error: {}".format(e)) - pass \ No newline at end of file + pass diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 7880d3f1a0e2..2e166ed0a222 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -82,7 +82,7 @@ def output_ports(self): """ return {"y_pred": NeuralType(('B', 'D'), ChannelType(), optional=True)} - def __init__(self, dim,name=None): + def __init__(self, dim, name=None): # Part specific for Neural Modules API: # (1) call base constructor # (2) define input and output ports diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 522cfad0fcba..a94984984d67 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -28,6 +28,7 @@ NeuralTypeComparisonResult, ) + class NeuralGraph(NeuralInterface): """ Neural Graph class stores dynamically defined graphs of connected Neural Modules. @@ -69,7 +70,6 @@ def __init__(self, operation_mode, name=None): self._modules = {} # "Steps": ordered execution of modules in a graph. self._steps = [] - def __call__(self, **kwargs): """ @@ -188,11 +188,11 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: Returns: A (dict) of module's output ports names to NeuralTypes mapping """ - #print("getter!") + # print("getter!") return self._bound_output_ports_default - #@output_ports.setter - #def output_ports(self, ports): + # @output_ports.setter + # def output_ports(self, ports): # print("setter!") # self._bound_output_ports_default = ports @@ -224,7 +224,6 @@ def deactivate(self): """ self._app_state.active_graph = None - def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) @@ -249,7 +248,7 @@ def __len__(self): def list_modules(self): desc = "{} ({}):\n".format(self.name, len(self)) for key, value in self._modules.items(): - desc += " * `{}` ({})\n".format(key, value ) + desc += " * `{}` ({})\n".format(key, value) return desc def record_step(self, module, inputs): @@ -257,7 +256,7 @@ def record_step(self, module, inputs): Records the operation (module plus passed inputs) on a list. """ # Check if module with that name already exists. - #if module.name in self._modules.keys(): + # if module.name in self._modules.keys(): # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) # Add module to list of modules. self._modules[module.name] = module diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph_manager.py index 2b3662cf94f4..8fc52397b7c7 100644 --- a/nemo/core/neural_graph_manager.py +++ b/nemo/core/neural_graph_manager.py @@ -16,10 +16,9 @@ # limitations under the License. # ============================================================================= -from nemo.utils.object_registry import ObjectRegistry - from nemo.core.neural_factory import OperationMode from nemo.core.neural_graph import NeuralGraph +from nemo.utils.object_registry import ObjectRegistry class NeuralGraphManager(ObjectRegistry): diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py index ec410c669ed9..5605feb1bc67 100644 --- a/nemo/core/neural_interface.py +++ b/nemo/core/neural_interface.py @@ -35,7 +35,7 @@ class NeuralInterface(ABC): def __init__(self, name): """ Constructor. Sets the application state. """ - # Copy the name. As names should be unique in module/graph scope, this should be handled additionally + # Copy the name. As names should be unique in module/graph scope, this should be handled additionally # in their constructors. self._name = name # Create access to the app state. @@ -73,4 +73,3 @@ def __call__(self, **kwargs): def name(self): """ Returns the object name. """ return self._name - diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 6eb5013ee8b3..953a1acfaa42 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -58,6 +58,7 @@ class WeightShareTransform(Enum): "PretrainedModleInfo", ("pretrained_model_name", "description", "parameters", "location"), ) + class NeuralModule(NeuralInterface): """ Abstract class that every Neural Module must inherit from. @@ -86,8 +87,6 @@ def __init__(self, name=None): # Optimization level. self._opt_level = self._factory.optim_level - - @property def init_params(self) -> Optional[Dict]: """ @@ -129,7 +128,6 @@ def __extract_init_params(self): # Return parameters. return init_params - def __validate_params(self, params): """ Checks whether dictionary contains parameters being primitive types (string, int, float etc.) @@ -413,7 +411,7 @@ def _disabled_deployment_output_ports(self) -> Optional[Set[str]]: """ return set([]) - def _prepare_for_deployment(self) -> None: + def _prepare_for_deployment(self) -> None: """Patch the module if required to prepare for deployment """ diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 4a7d96922d8a..c87ccdbd3313 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -17,6 +17,7 @@ from weakref import WeakSet + class ObjectRegistry(WeakSet): """ Registry used for storing references to objects, generating unique names and monitoring their `uniqueness`. @@ -64,7 +65,7 @@ def register(self, new_obj, name): # Return the name. return unique_name - + def __generate_unique_name(self): """ Generates a new unique name by adding postfix (number) to base name. diff --git a/tests/unit/utils/test_app_state.py b/tests/unit/utils/test_app_state.py index 5fcae6ee6d35..e872425cc012 100644 --- a/tests/unit/utils/test_app_state.py +++ b/tests/unit/utils/test_app_state.py @@ -20,7 +20,8 @@ from nemo.utils.app_state import AppState -class TestAppState(): + +class TestAppState: @pytest.mark.unit def test_value_sharing(self): # Create first instance of AppState. diff --git a/tests/unit/utils/test_object_registry.py b/tests/unit/utils/test_object_registry.py index 809e953e669c..e1f0e57709fb 100644 --- a/tests/unit/utils/test_object_registry.py +++ b/tests/unit/utils/test_object_registry.py @@ -20,8 +20,8 @@ from nemo.utils.object_registry import ObjectRegistry -class TestAppState(): +class TestAppState: @pytest.mark.unit def test_registry(self): """ Tests registry reference management. """ @@ -52,14 +52,14 @@ def __init__(self, name=None): assert len(registry) == 4 # Delete all objects - aside of reference! - del(c1) - del(c2) - del(c3) - del(c4) + del c1 + del c2 + del c3 + del c4 assert len(registry) == 1 with pytest.raises(KeyError): registry["c4"] # Delete the last object. - del(c1_ref) + del c1_ref assert len(registry) == 0 From 868ec8a193147133bb9d6e73cbc9e260332bf9da Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 8 Apr 2020 17:50:37 -0700 Subject: [PATCH 017/106] turning examples into unit/integration tests, small fixes here and there, work in progress Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0.py | 47 -------- .../graph_composition_integration_tests1_1.py | 51 --------- .../graph_composition_integration_tests1_2.py | 56 ---------- .../graph_composition_integration_tests1_3.py | 2 +- .../graph_composition_integration_tests5.py | 72 ------------ nemo/core/neural_graph.py | 5 +- tests/integration/core/test_neural_graph.py | 88 ++++++++++----- tests/unclassified/test_unclassified_tts.py | 2 +- tests/unit/core/test_neural_graphs.py | 105 ++++++++++++++++++ tests/unit/utils/test_object_registry.py | 4 +- 10 files changed, 175 insertions(+), 257 deletions(-) delete mode 100644 examples/start_here/graph_composition_integration_tests0.py delete mode 100644 examples/start_here/graph_composition_integration_tests1_1.py delete mode 100644 examples/start_here/graph_composition_integration_tests1_2.py delete mode 100644 examples/start_here/graph_composition_integration_tests5.py create mode 100644 tests/unit/core/test_neural_graphs.py diff --git a/examples/start_here/graph_composition_integration_tests0.py b/examples/start_here/graph_composition_integration_tests0.py deleted file mode 100644 index 6a5729609d43..000000000000 --- a/examples/start_here/graph_composition_integration_tests0.py +++ /dev/null @@ -1,47 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import nemo - -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -fx = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() - -logging.info( - "This example shows how one can build a `default` (implicit) graph." - F" This approach works for applications containing a single graph/" -) - -# This will create a default (implicit) graph: "training". -x, t = dl() -p = fx(x=x) -lss = loss(predictions=p, target=t) - - -# SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests1_1.py b/examples/start_here/graph_composition_integration_tests1_1.py deleted file mode 100644 index e915831faa93..000000000000 --- a/examples/start_here/graph_composition_integration_tests1_1.py +++ /dev/null @@ -1,51 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import nemo -from nemo.core import NeuralGraph, OperationMode - -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -m2 = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() - -logging.info( - "This example shows how one can build an `explicit` graph." - F"It also shows how to decouple graph instance creation from its activation." -) - -# Create the g0 graph. -g0 = NeuralGraph(operation_mode=OperationMode.training) - -# Activate the "g0 graph context" - all operations will be recorded to g0. -with g0: - x, t = dl() - p = m2(x=x) - lss = loss(predictions=p, target=t) - -# SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests1_2.py b/examples/start_here/graph_composition_integration_tests1_2.py deleted file mode 100644 index 1df2408d8ecd..000000000000 --- a/examples/start_here/graph_composition_integration_tests1_2.py +++ /dev/null @@ -1,56 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import nemo -from nemo.core import NeuralGraph, OperationMode - -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -m2 = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() - -logging.info( - "This example shows how one can build an `explicit` graph." - F"It also shows how to activate and deactivate the g0 context `manually`" -) - -# Create the g0 graph. -g0 = NeuralGraph(operation_mode=OperationMode.training) - -# Activate the "g0 graph context" "manually" - all steps will be recorded to g0. -g0.activate() - -# Define g0 - connections between the modules. -x, t = dl() -p = m2(x=x) -lss = loss(predictions=p, target=t) - -# Deactivate the "g0 graph context" (this is really optional, as long as there are no other steps to be recorded). -g0.deactivate() - -# SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests1_3.py b/examples/start_here/graph_composition_integration_tests1_3.py index 21e865b2de6d..04498285de57 100755 --- a/examples/start_here/graph_composition_integration_tests1_3.py +++ b/examples/start_here/graph_composition_integration_tests1_3.py @@ -76,7 +76,7 @@ def my_DAG(): lss = graph.output_ports ## Pros: functions are easy to understand -## Cons: function must return nxTensors, eg cannot create callbacks in them +## Cons: function must return nmTensors, eg cannot create callbacks in them ## need to return relevant tensors # SimpleLossLoggerCallback will print loss values to console. diff --git a/examples/start_here/graph_composition_integration_tests5.py b/examples/start_here/graph_composition_integration_tests5.py deleted file mode 100644 index 6fec1be463f3..000000000000 --- a/examples/start_here/graph_composition_integration_tests5.py +++ /dev/null @@ -1,72 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import torch - -import nemo -from nemo.core import NeuralGraph, OperationMode - -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl_training = nemo.tutorials.RealFunctionDataLayer(n=1000, batch_size=32, name="dl_training") -fx = nemo.tutorials.TaylorNet(dim=4, name="fx") -fx2 = nemo.tutorials.TaylorNet(dim=4) -fx3 = nemo.tutorials.TaylorNet(dim=4) -print(fx3.name) -loss = nemo.tutorials.MSELoss(name="loss") -loss2 = nemo.tutorials.MSELoss() - -logging.info("This example shows how one can access modules nested in a graph.") - -# Build the training graph. -with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: - # Bind the input. - _ = fx(x=trainable_module) - _ = fx2(x=trainable_module) - _ = fx3(x=trainable_module) - # All outputs will be bound by default. - -# Compose two graphs into final graph. -with NeuralGraph(operation_mode=OperationMode.training, name="training_graph") as training_graph: - # Take outputs from the training DL. - x, t = dl_training() - # Pass them to the trainable module. - p = trainable_module(x=x) - # Pass both of them to loss. - lss = loss(predictions=p, target=t) - lss2 = loss2(predictions=p, target=t) - - -print(trainable_module.list_modules()) - -print(training_graph.list_modules()) - -# Access modules. -dl_training_ref = training_graph["dl_training"] -fx_ref = training_graph["fx"] -loss_ref = training_graph["loss"] - -# Throws an exception. -try: - _ = training_graph["other_module"] -except KeyError as e: - print("Got error: {}".format(e)) - pass diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index a94984984d67..0c1687420842 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -20,6 +20,7 @@ from typing import Dict, Optional import nemo +from nemo.core import OperationMode from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import ( NeuralPortNameMismatchError, @@ -34,13 +35,13 @@ class NeuralGraph(NeuralInterface): Neural Graph class stores dynamically defined graphs of connected Neural Modules. """ - def __init__(self, operation_mode, name=None): + def __init__(self, operation_mode=OperationMode.both, name=None): """ Constructor. Initializes graph variables. Args: operation_mode: Graph operation mode, that will be propagated along modules during graph creation. - [training | eval] + [training | eval | both] (DEFAULT: both) name: Name of the graph (optional) """ # Initialize the inferface. diff --git a/tests/integration/core/test_neural_graph.py b/tests/integration/core/test_neural_graph.py index c425e458e869..a8d81ffc3e1d 100644 --- a/tests/integration/core/test_neural_graph.py +++ b/tests/integration/core/test_neural_graph.py @@ -17,42 +17,78 @@ # limitations under the License. # ============================================================================= -from unittest import TestCase - import pytest -import nemo - +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph, OperationMode +from nemo.backends.pytorch.actions import PtActions @pytest.mark.usefixtures("neural_factory") -class TestNeuralGraph(TestCase): +class TestNeuralGraph: + @pytest.mark.integration - def test_create_simple_graph(self): + def test_nm_tensors(self): + """ + Tests whether nmTensors are correct. + """ # Create modules. - dl = nemo.tutorials.RealFunctionDataLayer(n=100, batch_size=16) - fx = nemo.tutorials.TaylorNet(dim=4) - loss = nemo.tutorials.MSELoss() + data_source = RealFunctionDataLayer(n=100, batch_size=1) + trainable_module = TaylorNet(dim=4) + loss = MSELoss() # Create the graph by connnecting the modules. - x, y = dl() - y_pred = fx(x=x) - _ = loss(predictions=y_pred, target=y) - - @pytest.mark.integration - def test_simple_chain(self): - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=1) - trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - loss = nemo.backends.pytorch.tutorials.MSELoss() x, y = data_source() y_pred = trainable_module(x=x) loss_tensor = loss(predictions=y_pred, target=y) # check producers' bookkeeping - self.assertEqual(loss_tensor.producer, loss) - self.assertEqual(loss_tensor.producer_args, {"predictions": y_pred, "target": y}) - self.assertEqual(y_pred.producer, trainable_module) - self.assertEqual(y_pred.producer_args, {"x": x}) - self.assertEqual(y.producer, data_source) - self.assertEqual(y.producer_args, {}) - self.assertEqual(x.producer, data_source) - self.assertEqual(x.producer_args, {}) + assert loss_tensor.producer == loss + assert loss_tensor.producer_args == {"predictions": y_pred, "target": y} + assert y_pred.producer == trainable_module + assert y_pred.producer_args == {"x": x} + assert y.producer == data_source + assert y.producer_args == {} + assert x.producer == data_source + assert x.producer_args == {} + + @pytest.mark.integration + def test_implicit_default_graph(self): + """ Tests integration of a `default` (implicit) graph. """ + # Create modules. + dl = RealFunctionDataLayer(n=100, batch_size=4) + fx = TaylorNet(dim=4) + loss = MSELoss() + + # This will create a default (implicit) graph: "training". + x, t = dl() + p = fx(x=x) + lss = loss(predictions=p, target=t) + + # Instantiate an optimizer to perform the `train` action. + optimizer = PtActions() + # Invoke "train" action - perform single forward-backard step. + optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") + + + @pytest.mark.integration + def test_explicit_graph(self): + """ Tests integration of an `explicit` graph and decoupling of graph creation from its activation. """ + # Create modules. + dl = RealFunctionDataLayer(n=100, batch_size=4) + fx = TaylorNet(dim=4) + loss = MSELoss() + + # Create the g0 graph. + g0 = NeuralGraph() + + # Activate the "g0 graph context" - all operations will be recorded to g0. + with g0: + x, t = dl() + p = fx(x=x) + lss = loss(predictions=p, target=t) + + # Instantiate an optimizer to perform the `train` action. + optimizer = PtActions() + # Invoke "train" action - perform single forward-backard step. + optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") + diff --git a/tests/unclassified/test_unclassified_tts.py b/tests/unclassified/test_unclassified_tts.py index 91981f526164..514a296cf866 100644 --- a/tests/unclassified/test_unclassified_tts.py +++ b/tests/unclassified/test_unclassified_tts.py @@ -192,7 +192,7 @@ def test_waveglow_training(self): [loss_t], callbacks=[callback], optimizer="sgd", optimization_params={"num_epochs": 10, "lr": 0.0003}, ) - @pytest.mark.integration + @pytest.mark.unclassified def test_fastspeech(self): data_layer = nemo_asr.AudioToTextDataLayer( manifest_filepath=self.manifest_filepath, diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/test_neural_graphs.py new file mode 100644 index 000000000000..e8fc20cb9e5e --- /dev/null +++ b/tests/unit/core/test_neural_graphs.py @@ -0,0 +1,105 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + + +import pytest + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph, OperationMode +from nemo.core.neural_types import NeuralTypeComparisonResult + +@pytest.mark.usefixtures("neural_factory") +class TestNeuralGraphs: + + @pytest.mark.unit + def test_explicit_graph_with_activation(self): + """ + Tests initialization of an `explicit` graph and decoupling of graph creation from its activation. + Also tests modules access. + """ + # Create modules. + dl = RealFunctionDataLayer(n=100, batch_size=4, name="dl") + fx = TaylorNet(dim=4, name="fx") + loss = MSELoss(name="loss") + + # Create the g0 graph. + g0 = NeuralGraph() + + # Activate the "g0 graph context" - all operations will be recorded to g0. + with g0: + x, t = dl() + p = fx(x=x) + lss = loss(predictions=p, target=t) + + # Assert that there are 3 modules in the graph. + assert len(g0) == 3 + + # Test access modules. + assert g0["dl"] is dl + assert g0["fx"] is fx + assert g0["loss"] is loss + + with pytest.raises(KeyError): + g0["other_module"] + + @pytest.mark.unit + def test_explicit_graph_manual_activation(self): + """ Tests initialization of an `explicit` graph using `manual` activation. """ + # Create modules. + dl = RealFunctionDataLayer(n=100, batch_size=4) + fx = TaylorNet(dim=4) + + # Create the g0 graph. + g0 = NeuralGraph() + + # Activate the "g0 graph context" "manually" - all steps will be recorded to g0. + g0.activate() + + # Define g0 - connections between the modules. + x, t = dl() + p = fx(x=x) + + # Deactivate the "g0 graph context". + # Note that this is really optional, as long as there are no other steps to be recorded. + g0.deactivate() + + # Assert that there are 2 modules in the graph. + assert len(g0) == 2 + + + + + @pytest.mark.unit + def test_default_output_ports(self): + """ Tests automatic binding of default output ports. """ + dl = RealFunctionDataLayer(n=10000, batch_size=128) + m2 = TaylorNet(dim=4) + loss = MSELoss() + + with NeuralGraph(operation_mode=OperationMode.both) as g1: + x, t = dl() + p = m2(x=x) + + # Tests output ports. + assert len(g1.output_ports) == 3 + assert g1.output_ports["x"].compare(x) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y"].compare(t) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME + + diff --git a/tests/unit/utils/test_object_registry.py b/tests/unit/utils/test_object_registry.py index e1f0e57709fb..6bdb756d6e50 100644 --- a/tests/unit/utils/test_object_registry.py +++ b/tests/unit/utils/test_object_registry.py @@ -36,7 +36,7 @@ def __init__(self, name=None): # Test object uniqueness. c1 = MockupObjectClass("c1") c1_ref = registry["c1"] - assert c1_ref == c1 + assert c1_ref is c1 # Test name uniqueness. c2 = MockupObjectClass("c2") @@ -57,6 +57,8 @@ def __init__(self, name=None): del c3 del c4 assert len(registry) == 1 + # Assert that "c1" is still there, but "c4" is not. + registry["c1"] with pytest.raises(KeyError): registry["c4"] From 51e852ce19f4c2899f8575ab6a31968fdea276c6 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 09:57:14 -0700 Subject: [PATCH 018/106] Graph and module nesting - operation mode injected, implemented functionality with unit/integration tests Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 23 +++- nemo/core/neural_modules.py | 21 +++- tests/integration/core/test_neural_graph.py | 3 +- .../core/test_neural_graph_nesting.py | 100 ++++++++++++++++ tests/unit/core/test_neural_graph_nesting.py | 109 ++++++++++++++++++ 5 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 tests/integration/core/test_neural_graph_nesting.py create mode 100644 tests/unit/core/test_neural_graph_nesting.py diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 0c1687420842..582d6b080ff6 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -74,10 +74,26 @@ def __init__(self, operation_mode=OperationMode.both, name=None): def __call__(self, **kwargs): """ - This method "inserts" one existing neural graph into another one. + This method "nests" one existing neural graph into another one. Also checks if all inputs were provided and properly connects them. """ + # Test operation modes of the nested graphs. + outer_mode = self._app_state.active_graph.operation_mode + inner_mode = self.operation_mode + + if inner_mode == OperationMode.inference and outer_mode == OperationMode.training: + raise TypeError("Cannot nest 'inference' graph into 'training'") + + if inner_mode == OperationMode.training and outer_mode == OperationMode.inference: + raise TypeError("Cannot nest 'training' graph into 'inference'") + + if inner_mode == OperationMode.training and outer_mode == OperationMode.both: + raise TypeError("Cannot nest 'training' graph into 'both'") + + if inner_mode == OperationMode.inference and outer_mode == OperationMode.both: + raise TypeError("Cannot nest 'inference' graph into 'both'") + # print(" Neural Graph {} __call__".format(self._name)) # Get input and output ports definitions. input_port_defs = self.input_ports @@ -192,6 +208,11 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: # print("getter!") return self._bound_output_ports_default + @property + def operation_mode(self): + """ Returns operation mode. """ + return self._operation_mode + # @output_ports.setter # def output_ports(self, ports): # print("setter!") diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 953a1acfaa42..54ec4e983aef 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -39,7 +39,7 @@ NmTensor, ) from nemo import logging -from nemo.core import NeuralModuleFactory +from nemo.core import NeuralModuleFactory, OperationMode from nemo.core.neural_interface import NeuralInterface from nemo.package_info import __version__ as nemo_version from nemo.utils.decorators.deprecated import deprecated @@ -77,6 +77,9 @@ def __init__(self, name=None): # Register module and store the generated name. self._name = self._app_state.register_module(self, name) + # Set "both" as default operation mode. + self._operation_mode = OperationMode.both + # Get default factory. self._factory = NeuralModuleFactory.get_default_factory() @@ -417,6 +420,16 @@ def _prepare_for_deployment(self) -> None: """ return + @property + def operation_mode(self): + """ Returns the operation mode. """ + return self._operation_mode + + @operation_mode.setter + def operation_mode(self, operation_mode): + """ Sets the operation mode. """ + self._operation_mode = operation_mode + @staticmethod def pretrained_storage(): return '' @@ -439,11 +452,15 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ + # Set the operation mode of the outer graph. + self.operation_mode = self._app_state.active_graph.operation_mode + # print(" Neural Module:__call__") - # Get input and output ports definitions. + # Get input and output ports definitions - potentially depending on the operation mode! input_port_defs = self.input_ports output_port_defs = self.output_ports + # Record the operation (i.e. add a single module). self._app_state.active_graph.record_step(self, kwargs.items()) diff --git a/tests/integration/core/test_neural_graph.py b/tests/integration/core/test_neural_graph.py index a8d81ffc3e1d..8f15ba3068c2 100644 --- a/tests/integration/core/test_neural_graph.py +++ b/tests/integration/core/test_neural_graph.py @@ -1,6 +1,5 @@ # ! /usr/bin/python # -*- coding: utf-8 -*- - # ============================================================================= # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # @@ -20,7 +19,7 @@ import pytest from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph from nemo.backends.pytorch.actions import PtActions @pytest.mark.usefixtures("neural_factory") diff --git a/tests/integration/core/test_neural_graph_nesting.py b/tests/integration/core/test_neural_graph_nesting.py new file mode 100644 index 000000000000..e254072144af --- /dev/null +++ b/tests/integration/core/test_neural_graph_nesting.py @@ -0,0 +1,100 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest +import torch + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph, OperationMode, EvaluatorCallback, SimpleLossLoggerCallback +from nemo.backends.pytorch.actions import PtActions +from nemo.utils import logging + +@pytest.mark.usefixtures("neural_factory") +class TestNeuralGraphNesting: + + @pytest.mark.integration + def test_nesting_operation_modes_ok(self): + """ + Tests whether one can nest one graph in mode `both` (representing the our `model`) into + `training` and validation (`inference`) graphs. + """ + # Instantiate the necessary neural modules. + dl_training = RealFunctionDataLayer(n=100, batch_size=4) + dl_validation = RealFunctionDataLayer(n=100, batch_size=4) + fx = TaylorNet(dim=4) + loss = MSELoss() + + with NeuralGraph(operation_mode=OperationMode.both) as model: + # Bind the input. + _ = fx(x=model) + # All outputs will be bound by default. + + # Nest model into training graph. + with NeuralGraph(operation_mode=OperationMode.training) as training_graph: + # Take outputs from the training DL. + x, t = dl_training() + # Pass them to the model + p = model(x=x) + # Pass both of them to loss. + lss = loss(predictions=p, target=t) + + # Nest model into validation graph. + with NeuralGraph(operation_mode=OperationMode.inference) as validation_graph: + # Take outputs from the training DL. + x_valid, t_valid = dl_training() + # Pass them to the model + p_valid = model(x=x_valid) + loss_e = loss(predictions=p_valid, target=t_valid) + + + # Callbacks to print info to console and Tensorboard. + train_callback = SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') + ) + + + def batch_loss_per_batch_callback(tensors, global_vars): + if "batch_loss" not in global_vars.keys(): + global_vars["batch_loss"] = [] + for key, value in tensors.items(): + if key.startswith("loss"): + global_vars["batch_loss"].append(torch.mean(torch.stack(value))) + + + def batch_loss_epoch_finished_callback(global_vars): + epoch_loss = torch.max(torch.tensor(global_vars["batch_loss"])) + logging.info("Evaluation Loss: {0}".format(epoch_loss)) + return dict({"Evaluation Loss": epoch_loss}) + + + eval_callback = EvaluatorCallback( + eval_tensors=[loss_e], + user_iter_callback=batch_loss_per_batch_callback, + user_epochs_done_callback=batch_loss_epoch_finished_callback, + eval_step=1, + ) + + # Instantiate an optimizer to perform the `train` action. + optimizer = PtActions() + # Invoke "train" action - perform single forward-backard step. + optimizer.train([lss], + callbacks=[train_callback, eval_callback], + optimization_params={"max_steps": 2, "lr": 0.0003}, + optimizer="sgd", + ) diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py new file mode 100644 index 000000000000..98f1be54fe58 --- /dev/null +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -0,0 +1,109 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest +import torch + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph, OperationMode, EvaluatorCallback, SimpleLossLoggerCallback +from nemo.backends.pytorch.actions import PtActions +from nemo.utils import logging + +@pytest.mark.usefixtures("neural_factory") +class TestNeuralGraphNesting: + + @pytest.mark.unit + def test_module_nesting_change_operation_modes(self): + """ + Tests whether invalid nesting (i.e. nesting of graphs with incompatible modes) throw exeptions. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=4) + + with NeuralGraph(operation_mode=OperationMode.both): + _, _ = dl() + assert dl.operation_mode == OperationMode.both + + with NeuralGraph(operation_mode=OperationMode.training): + _, _ = dl() + assert dl.operation_mode == OperationMode.training + + with NeuralGraph(operation_mode=OperationMode.inference): + _, _ = dl() + assert dl.operation_mode == OperationMode.inference + + + @pytest.mark.unit + def test_graph_nesting_possible_operation_modes(self): + """ + Tests whether invalid nesting (i.e. nesting of graphs with incompatible modes) throw exeptions. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=4) + + with NeuralGraph(operation_mode=OperationMode.both) as both: + _, _ = dl() + + with NeuralGraph(operation_mode=OperationMode.training) as training: + _, _ = dl() + + with NeuralGraph(operation_mode=OperationMode.inference) as inference: + _, _ = dl() + + # Allowed operations. + # Can nest 'both' into 'training'. + with NeuralGraph(operation_mode=OperationMode.training): + _, _ = both() + + # Can nest 'both' into 'inference'. + with NeuralGraph(operation_mode=OperationMode.inference): + _, _ = both() + + # Can nest 'training' into 'training'. + with NeuralGraph(operation_mode=OperationMode.training): + _, _ = training() + + # Can nest 'inference' into 'inference'. + with NeuralGraph(operation_mode=OperationMode.inference): + _, _ = inference() + + # Can nest 'both' into 'both'. + with NeuralGraph(operation_mode=OperationMode.both): + _, _ = both() + + # Operations not allowed. + # Cannot nest 'inference' into 'training'. + with pytest.raises(TypeError): + with NeuralGraph(operation_mode=OperationMode.training): + _, _ = inference() + + # Cannot nest 'training' into 'inference'. + with pytest.raises(TypeError): + with NeuralGraph(operation_mode=OperationMode.inference): + _, _ = training() + + # Cannot nest 'training' into 'both'. + with pytest.raises(TypeError): + with NeuralGraph(operation_mode=OperationMode.both): + _, _ = training() + + # Cannot nest 'inference' into 'both'. + with pytest.raises(TypeError): + with NeuralGraph(operation_mode=OperationMode.both): + _, _ = inference() From e2ed33af8c954058ff2693d1c7354f129cd0f8c8 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 09:59:15 -0700 Subject: [PATCH 019/106] formatting fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_modules.py | 1 - tests/integration/core/test_neural_graph.py | 6 ++---- tests/integration/core/test_neural_graph_nesting.py | 13 +++++-------- tests/unit/core/test_neural_graph_nesting.py | 7 +++---- tests/unit/core/test_neural_graphs.py | 9 ++------- 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 54ec4e983aef..08e7d00fa880 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -460,7 +460,6 @@ def __call__(self, **kwargs): input_port_defs = self.input_ports output_port_defs = self.output_ports - # Record the operation (i.e. add a single module). self._app_state.active_graph.record_step(self, kwargs.items()) diff --git a/tests/integration/core/test_neural_graph.py b/tests/integration/core/test_neural_graph.py index 8f15ba3068c2..df74c446a8be 100644 --- a/tests/integration/core/test_neural_graph.py +++ b/tests/integration/core/test_neural_graph.py @@ -18,13 +18,13 @@ import pytest +from nemo.backends.pytorch.actions import PtActions from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import NeuralGraph -from nemo.backends.pytorch.actions import PtActions + @pytest.mark.usefixtures("neural_factory") class TestNeuralGraph: - @pytest.mark.integration def test_nm_tensors(self): """ @@ -68,7 +68,6 @@ def test_implicit_default_graph(self): # Invoke "train" action - perform single forward-backard step. optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") - @pytest.mark.integration def test_explicit_graph(self): """ Tests integration of an `explicit` graph and decoupling of graph creation from its activation. """ @@ -90,4 +89,3 @@ def test_explicit_graph(self): optimizer = PtActions() # Invoke "train" action - perform single forward-backard step. optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") - diff --git a/tests/integration/core/test_neural_graph_nesting.py b/tests/integration/core/test_neural_graph_nesting.py index e254072144af..5b9b4e85a6a2 100644 --- a/tests/integration/core/test_neural_graph_nesting.py +++ b/tests/integration/core/test_neural_graph_nesting.py @@ -20,14 +20,14 @@ import pytest import torch -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import NeuralGraph, OperationMode, EvaluatorCallback, SimpleLossLoggerCallback from nemo.backends.pytorch.actions import PtActions +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging + @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphNesting: - @pytest.mark.integration def test_nesting_operation_modes_ok(self): """ @@ -62,13 +62,11 @@ def test_nesting_operation_modes_ok(self): p_valid = model(x=x_valid) loss_e = loss(predictions=p_valid, target=t_valid) - # Callbacks to print info to console and Tensorboard. train_callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') ) - def batch_loss_per_batch_callback(tensors, global_vars): if "batch_loss" not in global_vars.keys(): global_vars["batch_loss"] = [] @@ -76,13 +74,11 @@ def batch_loss_per_batch_callback(tensors, global_vars): if key.startswith("loss"): global_vars["batch_loss"].append(torch.mean(torch.stack(value))) - def batch_loss_epoch_finished_callback(global_vars): epoch_loss = torch.max(torch.tensor(global_vars["batch_loss"])) logging.info("Evaluation Loss: {0}".format(epoch_loss)) return dict({"Evaluation Loss": epoch_loss}) - eval_callback = EvaluatorCallback( eval_tensors=[loss_e], user_iter_callback=batch_loss_per_batch_callback, @@ -93,7 +89,8 @@ def batch_loss_epoch_finished_callback(global_vars): # Instantiate an optimizer to perform the `train` action. optimizer = PtActions() # Invoke "train" action - perform single forward-backard step. - optimizer.train([lss], + optimizer.train( + [lss], callbacks=[train_callback, eval_callback], optimization_params={"max_steps": 2, "lr": 0.0003}, optimizer="sgd", diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 98f1be54fe58..8b9c39a27a01 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -20,14 +20,14 @@ import pytest import torch -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import NeuralGraph, OperationMode, EvaluatorCallback, SimpleLossLoggerCallback from nemo.backends.pytorch.actions import PtActions +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging + @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphNesting: - @pytest.mark.unit def test_module_nesting_change_operation_modes(self): """ @@ -48,7 +48,6 @@ def test_module_nesting_change_operation_modes(self): _, _ = dl() assert dl.operation_mode == OperationMode.inference - @pytest.mark.unit def test_graph_nesting_possible_operation_modes(self): """ diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/test_neural_graphs.py index e8fc20cb9e5e..003ec2941a63 100644 --- a/tests/unit/core/test_neural_graphs.py +++ b/tests/unit/core/test_neural_graphs.py @@ -24,9 +24,9 @@ from nemo.core import NeuralGraph, OperationMode from nemo.core.neural_types import NeuralTypeComparisonResult + @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphs: - @pytest.mark.unit def test_explicit_graph_with_activation(self): """ @@ -49,7 +49,7 @@ def test_explicit_graph_with_activation(self): # Assert that there are 3 modules in the graph. assert len(g0) == 3 - + # Test access modules. assert g0["dl"] is dl assert g0["fx"] is fx @@ -82,9 +82,6 @@ def test_explicit_graph_manual_activation(self): # Assert that there are 2 modules in the graph. assert len(g0) == 2 - - - @pytest.mark.unit def test_default_output_ports(self): """ Tests automatic binding of default output ports. """ @@ -101,5 +98,3 @@ def test_default_output_ports(self): assert g1.output_ports["x"].compare(x) == NeuralTypeComparisonResult.SAME assert g1.output_ports["y"].compare(t) == NeuralTypeComparisonResult.SAME assert g1.output_ports["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME - - From a054311e691c8cb9b15fd74965bb4fee6e880b75 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 10:51:17 -0700 Subject: [PATCH 020/106] cleanup of output port logic in __call__, other cleanups, LGTM fixes Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 8 +- .../graph_composition_integration_tests1_3.py | 88 ------------------- nemo/backends/pytorch/tutorials/toys.py | 51 ----------- nemo/core/neural_modules.py | 30 ++----- 4 files changed, 8 insertions(+), 169 deletions(-) delete mode 100755 examples/start_here/graph_composition_integration_tests1_3.py diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 9e7e0025f52b..7759472ea227 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -30,10 +30,6 @@ logging = nemo.logging nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -fx = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() logging.info( "This example shows how one can build a Jasper model using the `default` (implicit) graph." @@ -70,10 +66,10 @@ greedy_decoder = nemo_asr.GreedyCTCDecoder() # Create the Jasper composite module. -with NeuralGraph(operation_mode=OperationMode.training) as Jasper: +with NeuralGraph() as Jasper: processed_signal, processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=processed_signal_len) - log_probs = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) + _ = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) # Create the "implicit" training graph. audio_signal, audio_signal_len, transcript, transcript_len = data_layer() diff --git a/examples/start_here/graph_composition_integration_tests1_3.py b/examples/start_here/graph_composition_integration_tests1_3.py deleted file mode 100755 index 04498285de57..000000000000 --- a/examples/start_here/graph_composition_integration_tests1_3.py +++ /dev/null @@ -1,88 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import nemo -from nemo.core import NeuralGraph, OperationMode -from nemo.utils.app_state import AppState - -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() -# Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -m2 = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() - -logging.info( - "This example shows how one can build an `explicit` graph." - F"It also shows how to activate and deactivate the g0 context `manually`" -) - - -def NeuralGraphDecorator(func): - def wrapper(*args, **kwargs): - # Create the g0 graph. - g0 = NeuralGraph(operation_mode=OperationMode.training) - - # Activate the "g0 graph context" "manually" - all operations will be recorded to g0. - g0.activate() - - # Extract input_ports - input_ports = list(args) - for key, value in kwargs.items(): - input_ports.append(value) - - # Run user-defined function - output_ports = func(*args, **kwargs) - - # Record ports - g0.input_ports = input_ports - g0.output_ports = output_ports - - # Deactivate the "g0 graph context" (this is really optional, as long as there are no other operations). - g0.deactive() - - # Return our new compose neural module - return g0 - - return wrapper - - -@NeuralGraphDecorator -def my_DAG(): - x, t = dl() - p = m2(x=x) - lss = loss(predictions=p, target=t) - return lss - - -graph = my_DAG() -lss = graph.output_ports - -## Pros: functions are easy to understand -## Cons: function must return nmTensors, eg cannot create callbacks in them -## need to return relevant tensors - -# SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 2e166ed0a222..4a99c7acdb46 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -7,7 +7,6 @@ from nemo import logging from nemo.backends.pytorch.nm import DataLayerNM, LossNM, TrainableNM -from nemo.core import DeviceType, NeuralModule from nemo.core.neural_types import * from nemo.utils.decorators import add_port_docs @@ -49,7 +48,6 @@ def __init__(self, dim, name=None): self._dim = dim self.fc1 = nn.Linear(self._dim, 1) t.nn.init.xavier_uniform_(self.fc1.weight) - self._device = t.device("cuda" if self.placement == DeviceType.GPU else "cpu") self.to(self._device) # IMPORTANT: input arguments to forward must match input input ports' names @@ -61,54 +59,6 @@ def forward(self, x): return self.fc1(nx) -class TaylorNetO(TrainableNM): # Note inheritance from TrainableNM - """Module which learns Taylor's coefficients.""" - - @property - @add_port_docs() - def input_ports(self): - """Returns definitions of module input ports. - - """ - return { - "x": NeuralType(('B', 'D'), ChannelType()), - "o": NeuralType(('B', 'D'), ChannelType()), - } - - @property - @add_port_docs() - def output_ports(self): - """Returns definitions of module output ports. - """ - return {"y_pred": NeuralType(('B', 'D'), ChannelType(), optional=True)} - - def __init__(self, dim, name=None): - # Part specific for Neural Modules API: - # (1) call base constructor - # (2) define input and output ports - super().__init__(name=name) - - # And of Neural Modules specific part. Rest is Pytorch code - self._dim = dim - self.fc1 = nn.Linear(self._dim, 1) - t.nn.init.xavier_uniform_(self.fc1.weight) - self._device = t.device("cuda" if self.placement == DeviceType.GPU else "cpu") - self.to(self._device) - - # IMPORTANT: input arguments to forward must match input input ports' names - # If port is Optional, the default value should be None - def forward(self, x, o=None): - lst = [] - if o is None: - logging.debug("O is None") - else: - logging.debug("O is not None") - for pw in range(self._dim): - lst.append(x ** pw) - nx = t.cat(lst, dim=-1) - return self.fc1(nx) - - class RealFunctionDataLayer(DataLayerNM): """ Data layer that yields (x, f(x)) data and label pairs. @@ -159,7 +109,6 @@ def __init__(self, batch_size, f_name="sin", n=1000, x_lo=-4, x_hi=4, name=None) self._n = n self._batch_size = batch_size - self._device = t.device("cuda" if self.placement == DeviceType.GPU else "cpu") x_data = t.tensor(np.random.uniform(low=x_lo, high=x_hi, size=self._n)).unsqueeze(-1).to(self._device) y_data = func(x_data) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 08e7d00fa880..334e4bc09d24 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -463,8 +463,6 @@ def __call__(self, **kwargs): # Record the operation (i.e. add a single module). self._app_state.active_graph.record_step(self, kwargs.items()) - first_input_nmtensor_type = None - input_nmtensors_are_of_same_type = True # Iterate through all passed parameters. for port_name, port_content in kwargs.items(): # make sure that passed arguments correspond to input port names @@ -500,7 +498,6 @@ def __call__(self, **kwargs): type_comatibility, ) ) - # TODO CHECK 1: Are we making sure that ALL necessary inputs that were PASSED? # Here we will store the results. results = None @@ -508,31 +505,16 @@ def __call__(self, **kwargs): if len(output_port_defs) == 1: out_name = list(output_port_defs)[0] out_type = output_port_defs[out_name] - if out_type is None: - if input_nmtensors_are_of_same_type: - out_type = first_input_nmtensor_type - else: - raise CanNotInferResultNeuralType( - "Can't infer output neural type. Likely your inputs are of different type." - ) - # TODO CHECK 2: Why are we returning "something" (having input type) if there SHOULD be NO output? + results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) # Bind the output ports. self._app_state.active_graph.bind_outputs(output_port_defs, [results]) else: - result = [] - for out_port, n_type in output_port_defs.items(): - out_type = n_type - if out_type is None: - if input_nmtensors_are_of_same_type: - out_type = first_input_nmtensor_type - else: - raise CanNotInferResultNeuralType( - "Can't infer output neural type. Likely your inputs are of different type." - ) - result.append(NmTensor(producer=self, producer_args=kwargs, name=out_port, ntype=out_type,)) + results_list = [] + for out_name, out_type in output_port_defs.items(): + results_list.append(NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,)) # Creating ad-hoc class for returning from module's forward pass. output_class_name = f'{self.__class__.__name__}Output' @@ -540,10 +522,10 @@ def __call__(self, **kwargs): result_type = collections.namedtuple(typename=output_class_name, field_names=field_names,) # Bind the output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, result) + self._app_state.active_graph.bind_outputs(output_port_defs, results_list) # Tie tuple of output tensors with corresponding names. - results = result_type(*result) + results = result_type(*results_list) # Return the results. return results From c0e5f0b0a52da4d66a2f65ad74e19d5b6245ac17 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 11:01:49 -0700 Subject: [PATCH 021/106] Further cleanup of neural_modules.py Signed-off-by: Tomasz Kornuta --- nemo/core/neural_modules.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 334e4bc09d24..71bf9518efde 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -29,8 +29,7 @@ from ruamel.yaml import YAML -import nemo -from .neural_types import ( +from nemo.core.neural_types import ( CanNotInferResultNeuralType, NeuralPortNameMismatchError, NeuralPortNmTensorMismatchError, @@ -38,8 +37,8 @@ NeuralTypeComparisonResult, NmTensor, ) -from nemo import logging -from nemo.core import NeuralModuleFactory, OperationMode +from nemo.utils import logging +from nemo.core import NeuralModuleFactory, OperationMode, NeuralGraph from nemo.core.neural_interface import NeuralInterface from nemo.package_info import __version__ as nemo_version from nemo.utils.decorators.deprecated import deprecated @@ -452,6 +451,7 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ + # Set the operation mode of the outer graph. self.operation_mode = self._app_state.active_graph.operation_mode @@ -470,7 +470,7 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) # Check what was actually passed. - if isinstance(port_content, nemo.core.NeuralGraph): + if isinstance(port_content, NeuralGraph): # Bind this input port to a neural graph. # TODO: make sure that port_content == self._app_state.active_graph ????? From 98cfe434a6e455296f432454f4836e7aa5ba2f8c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 11:19:19 -0700 Subject: [PATCH 022/106] LGTM cleanups Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0_jasper.py | 2 +- nemo/core/neural_modules.py | 1 - nemo/utils/object_registry.py | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 7759472ea227..0fab64908815 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -25,7 +25,7 @@ import nemo import nemo.collections.asr as nemo_asr from nemo.collections.asr.helpers import monitor_asr_train_progress -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph logging = nemo.logging diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 71bf9518efde..0644004ea62c 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -30,7 +30,6 @@ from ruamel.yaml import YAML from nemo.core.neural_types import ( - CanNotInferResultNeuralType, NeuralPortNameMismatchError, NeuralPortNmTensorMismatchError, NeuralType, diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index c87ccdbd3313..49b909a20b5b 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -107,3 +107,9 @@ def __getitem__(self, key): return obj # Else: seems that there is no object with that name. raise KeyError("A {} with name `{}` don't exists!".format(self._base_type_name, key)) + + def __eq__(self, other): + """ Checks if two resitrys have similar content. """ + if not isinstance(other, WeakSet): + return False + return super().__eq__(other) \ No newline at end of file From d109b5c74e82e892658cbd6d8b8a49b861fdf1cb Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 11:22:06 -0700 Subject: [PATCH 023/106] style fix:] Signed-off-by: Tomasz Kornuta --- nemo/core/neural_modules.py | 6 +++--- nemo/utils/object_registry.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 0644004ea62c..d04fc05e2b2d 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -29,6 +29,8 @@ from ruamel.yaml import YAML +from nemo.core import NeuralGraph, NeuralModuleFactory, OperationMode +from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import ( NeuralPortNameMismatchError, NeuralPortNmTensorMismatchError, @@ -36,10 +38,8 @@ NeuralTypeComparisonResult, NmTensor, ) -from nemo.utils import logging -from nemo.core import NeuralModuleFactory, OperationMode, NeuralGraph -from nemo.core.neural_interface import NeuralInterface from nemo.package_info import __version__ as nemo_version +from nemo.utils import logging from nemo.utils.decorators.deprecated import deprecated YAML = YAML(typ='safe') diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 49b909a20b5b..adfcec7041c3 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -112,4 +112,4 @@ def __eq__(self, other): """ Checks if two resitrys have similar content. """ if not isinstance(other, WeakSet): return False - return super().__eq__(other) \ No newline at end of file + return super().__eq__(other) From 6487a654b15f08a87483c8da1b09771af596cfb2 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 11:57:09 -0700 Subject: [PATCH 024/106] micro cleanup Signed-off-by: Tomasz Kornuta --- nemo/core/neural_interface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py index 5605feb1bc67..b1f3219b2f1f 100644 --- a/nemo/core/neural_interface.py +++ b/nemo/core/neural_interface.py @@ -20,7 +20,6 @@ from typing import Dict, Optional import nemo -from nemo.core.neural_types import NeuralType class NeuralInterface(ABC): @@ -43,7 +42,7 @@ def __init__(self, name): @property @abstractmethod - def input_ports(self) -> Optional[Dict[str, NeuralType]]: + def input_ports(self) -> Optional[Dict[str, nemo.core.neural_types.NeuralType]]: """ Returns definitions of module input ports Returns: @@ -52,7 +51,7 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: @property @abstractmethod - def output_ports(self) -> Optional[Dict[str, NeuralType]]: + def output_ports(self) -> Optional[Dict[str, nemo.core.neural_types.NeuralType]]: """ Returns definitions of module output ports Returns: From 6c16c6e96b816e80b9bdac38e91f14d63e8b6851 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 11:58:48 -0700 Subject: [PATCH 025/106] micro cleanup2 Signed-off-by: Tomasz Kornuta --- nemo/core/neural_interface.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py index b1f3219b2f1f..5605feb1bc67 100644 --- a/nemo/core/neural_interface.py +++ b/nemo/core/neural_interface.py @@ -20,6 +20,7 @@ from typing import Dict, Optional import nemo +from nemo.core.neural_types import NeuralType class NeuralInterface(ABC): @@ -42,7 +43,7 @@ def __init__(self, name): @property @abstractmethod - def input_ports(self) -> Optional[Dict[str, nemo.core.neural_types.NeuralType]]: + def input_ports(self) -> Optional[Dict[str, NeuralType]]: """ Returns definitions of module input ports Returns: @@ -51,7 +52,7 @@ def input_ports(self) -> Optional[Dict[str, nemo.core.neural_types.NeuralType]]: @property @abstractmethod - def output_ports(self) -> Optional[Dict[str, nemo.core.neural_types.NeuralType]]: + def output_ports(self) -> Optional[Dict[str, NeuralType]]: """ Returns definitions of module output ports Returns: From b7d3f1cdaae597c79466a1b4399300e40cdbe6cf Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 9 Apr 2020 13:05:01 -0700 Subject: [PATCH 026/106] punctuation module name fix Signed-off-by: Tomasz Kornuta --- .../nlp/nm/trainables/common/token_classification_nm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index 0561321781f7..e101b411a5ee 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -132,7 +132,7 @@ def __init__( use_transformer_pretrained=True, ): # Pass name up the module class hierarchy. - super().__init__(name) + super().__init__(name=name) self.mlp = MultiLayerPerceptron(hidden_size, num_classes, self._device, num_layers, activation, log_softmax) self.dropout = nn.Dropout(dropout) From 5548495de9b303d92b6da09e02475f4af0d68bc9 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 10:40:07 -0700 Subject: [PATCH 027/106] Working on NmTensor extensions Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests1.py | 5 + nemo/core/neural_modules.py | 2 + nemo/core/neural_types/neural_type.py | 48 ++++++- tests/integration/core/test_neural_graph.py | 25 ---- tests/unit/core/test_nm_tensor.py | 123 ++++++++++++++++++ 5 files changed, 175 insertions(+), 28 deletions(-) create mode 100644 tests/unit/core/test_nm_tensor.py diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index cb941f921d97..e46028aa9c57 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -34,6 +34,11 @@ x, t = dl() p = m2(x=x) lss = loss(predictions=p, target=t) + # Manual bind. + g0.output_ports["output"] = loss + +print(g0.output_ports) +print(g0.output_ports["x"]) # SimpleLossLoggerCallback will print loss values to console. callback = nemo.core.SimpleLossLoggerCallback( diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index d04fc05e2b2d..d3e2f71c4816 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -497,6 +497,8 @@ def __call__(self, **kwargs): type_comatibility, ) ) + # Ok, we have checked the input, let's "consume" it. + port_content.add_consumer(self, port_name) # Here we will store the results. results = None diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index ad38bc290859..0df83c105fc1 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -187,6 +187,9 @@ def __compare_axes(axes_a, axes_b) -> int: return 3 +from collections import namedtuple +ModulePort = namedtuple('ModulePort', ["name", "port"]) + class NmTensor(NeuralType): """Class representing data which flows between NeuralModules' ports. It also has a type of NeuralType represented by inheriting from NeuralType @@ -203,8 +206,10 @@ def __init__(self, producer, producer_args, name, ntype=None): super(NmTensor, self).__init__(axes=ntype.axes, elements_type=ntype.elements_type, optional=ntype.optional) self._producer = producer self._producer_args = producer_args - self._name = name + self._output_port_name = name self._uuid = str(uuid.uuid4()) + # List of tuples (consumer name, input port name) + self._consumers_ports = [] @property def producer(self): @@ -214,6 +219,43 @@ def producer(self): """ return self._producer + @property + def producer_port(self): + """ + Returns: + A tuple containing producer name and corresponding output port name. + """ + return ModulePort(self._producer.name, self._output_port_name) + + @property + def consumers_ports(self): + """ + Returns: + A list of tuples containing consumer name and corresponding input port names. + """ + return self._consumers_ports + + def add_consumer(self, module, input_port_name): + """ + Adds tensor "consumer". + + Args: + module: Module that accepts the tensor as input. + input_port_name: Name of the module's input port. + + """ + self._consumers_ports.append(ModulePort(module.name, input_port_name)) + + + @property + def type(self): + """ + Returns: + Neural Type associated with this NmTensor. + """ + return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) + + @property def producer_args(self): """ @@ -230,7 +272,7 @@ def name(self): A NmTensor's name which should be equal to the NeuralModule's output port's name which created it """ - return self._name + return self._output_port_name @property def unique_name(self): @@ -243,7 +285,7 @@ def unique_name(self): """ if self._producer is None: raise ValueError("This NmTensor does not have a unique name") - return f"{self._name}~~~{self.producer}~~~{self._uuid}" + return f"{self._output_port_name}~~~{self.producer.name}~~~{self._uuid}" class NeuralTypeError(Exception): diff --git a/tests/integration/core/test_neural_graph.py b/tests/integration/core/test_neural_graph.py index df74c446a8be..210921c5f848 100644 --- a/tests/integration/core/test_neural_graph.py +++ b/tests/integration/core/test_neural_graph.py @@ -25,31 +25,6 @@ @pytest.mark.usefixtures("neural_factory") class TestNeuralGraph: - @pytest.mark.integration - def test_nm_tensors(self): - """ - Tests whether nmTensors are correct. - """ - # Create modules. - data_source = RealFunctionDataLayer(n=100, batch_size=1) - trainable_module = TaylorNet(dim=4) - loss = MSELoss() - - # Create the graph by connnecting the modules. - x, y = data_source() - y_pred = trainable_module(x=x) - loss_tensor = loss(predictions=y_pred, target=y) - - # check producers' bookkeeping - assert loss_tensor.producer == loss - assert loss_tensor.producer_args == {"predictions": y_pred, "target": y} - assert y_pred.producer == trainable_module - assert y_pred.producer_args == {"x": x} - assert y.producer == data_source - assert y.producer_args == {} - assert x.producer == data_source - assert x.producer_args == {} - @pytest.mark.integration def test_implicit_default_graph(self): """ Tests integration of a `default` (implicit) graph. """ diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py new file mode 100644 index 000000000000..1b121cf978a8 --- /dev/null +++ b/tests/unit/core/test_nm_tensor.py @@ -0,0 +1,123 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core.neural_types import NeuralTypeComparisonResult + + +@pytest.mark.usefixtures("neural_factory") +class TestNmTensor: + @pytest.mark.unit + def test_nm_tensors_producer_args(self): + """ + Tests whether nmTensors are correct - producers and their args. + """ + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1) + trainable_module = TaylorNet(dim=4) + loss = MSELoss() + + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = trainable_module(x=x) + loss_tensor = loss(predictions=y_pred, target=y) + + # check producers' bookkeeping + assert loss_tensor.producer == loss + assert loss_tensor.producer_args == {"predictions": y_pred, "target": y} + assert y_pred.producer is trainable_module + assert y_pred.producer_args == {"x": x} + assert y.producer is data_source + assert y.producer_args == {} + assert x.producer is data_source + assert x.producer_args == {} + + + @pytest.mark.unit + def test_nm_tensors_producer_consumers(self): + """ + Tests whether nmTensors are correct - checking producers and consumers. + """ + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1, name="source") + trainable_module = TaylorNet(dim=4, name="tm") + loss = MSELoss(name="loss") + loss2 = MSELoss(name="loss2") + + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = trainable_module(x=x) + lss = loss(predictions=y_pred, target=y) + lss2 = loss2(predictions=y_pred, target=y) + + # Check tensor x producer and consumers. + p = x.producer_port + cs = x.consumers_ports + assert p.name == "source" + assert p.port == "x" + assert len(cs) == 1 + assert cs[0].name == "tm" + assert cs[0].port == "x" + + # Check tensor y producer and consumers. + p = y.producer_port + cs = y.consumers_ports + assert p.name == "source" + assert p.port == "y" + assert len(cs) == 2 + assert cs[0].name == "loss" + assert cs[0].port == "target" + assert cs[1].name == "loss2" + assert cs[1].port == "target" + + # Check tensor y_pred producer and consumers. + p = y_pred.producer_port + cs = y_pred.consumers_ports + assert p.name == "tm" + assert p.port == "y_pred" + assert len(cs) == 2 + assert cs[0].name == "loss" + assert cs[0].port == "predictions" + assert cs[1].name == "loss2" + assert cs[1].port == "predictions" + + + @pytest.mark.unit + def test_nm_tensors_types(self): + """ + Tests whether nmTensors are correct - checking type property. + """ + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1) + trainable_module = TaylorNet(dim=4) + loss = MSELoss() + + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = trainable_module(x=x) + lss = loss(predictions=y_pred, target=y) + + # Check types. + assert x.type.compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert y.type.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert y_pred.type.compare(trainable_module.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert lss.type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + From c835279161c3eb6f78e0331e903c2dae005dd1bd Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 10:40:48 -0700 Subject: [PATCH 028/106] style fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_types/neural_type.py | 5 ++--- tests/unit/core/test_nm_tensor.py | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 0df83c105fc1..b90dc4050c3a 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -24,6 +24,7 @@ 'CanNotInferResultNeuralType', ] import uuid +from collections import namedtuple from typing import Optional, Tuple from nemo.core.neural_types.axes import AxisKind, AxisType @@ -187,9 +188,9 @@ def __compare_axes(axes_a, axes_b) -> int: return 3 -from collections import namedtuple ModulePort = namedtuple('ModulePort', ["name", "port"]) + class NmTensor(NeuralType): """Class representing data which flows between NeuralModules' ports. It also has a type of NeuralType represented by inheriting from NeuralType @@ -246,7 +247,6 @@ def add_consumer(self, module, input_port_name): """ self._consumers_ports.append(ModulePort(module.name, input_port_name)) - @property def type(self): """ @@ -255,7 +255,6 @@ def type(self): """ return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) - @property def producer_args(self): """ diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 1b121cf978a8..3b54bb790b66 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -49,7 +49,6 @@ def test_nm_tensors_producer_args(self): assert x.producer is data_source assert x.producer_args == {} - @pytest.mark.unit def test_nm_tensors_producer_consumers(self): """ @@ -98,7 +97,6 @@ def test_nm_tensors_producer_consumers(self): assert cs[1].name == "loss2" assert cs[1].port == "predictions" - @pytest.mark.unit def test_nm_tensors_types(self): """ @@ -119,5 +117,3 @@ def test_nm_tensors_types(self): assert y.type.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME assert y_pred.type.compare(trainable_module.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert lss.type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - - From 70320fe6feb550cc0515e51992e49c544ed79f02 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 11:46:42 -0700 Subject: [PATCH 029/106] Cleaned up the call output tensor logic in neural module and graph classes Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 36 +++++++-------------------- nemo/core/neural_modules.py | 27 ++++++++++---------- nemo/core/neural_types/neural_type.py | 16 ++++++++++++ tests/system/test_pytorch_trainers.py | 9 ++----- tests/unit/core/test_nm_tensor.py | 18 ++++++++++++-- 5 files changed, 56 insertions(+), 50 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 582d6b080ff6..fa5c1ac39983 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -154,37 +154,19 @@ def __call__(self, **kwargs): # that will be used during graph backward traverse. output_tensor.producer_args[port_name] = port_content - # TODO CHECK 1: Are we making sure that ALL necessary inputs that were PASSED? - - # Here we will store the results. - results = None - - # This part is different. Now the goal is not to create NEW "tensors", but to return the bound ones! + # Create the module outputs. + # This part is different from Neural Module. + # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! if len(output_port_defs) == 1: - # Get the name of the ouput port. - out_name = list(output_port_defs)[0] - # Simply pass the bound tensor. - results = self._bound_output_tensors_default[out_name] - # BUT UPDATE THE inputs to it!! - - # Bind the output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, [results]) - + # Return the single tensor. + results = next(iter(self._bound_output_tensors_default.values())) else: - result = [] - for _, tensor in self._bound_output_tensors_default.items(): - result.append(tensor) - - # Creating ad-hoc class for returning from module's forward pass. + # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' - field_names = list(output_port_defs) - result_type = collections.namedtuple(typename=output_class_name, field_names=field_names,) - - # Bind the output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, result) + result_type = namedtuple(typename=output_class_name, field_names=output_port_defs.keys()) - # Tie tuple of output tensors with corresponding names. - results = result_type(*result) + # Bind the "default" output ports. + results = result_type(*self._bound_output_tensors_default) # Return the results. return results diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index d3e2f71c4816..c2db6d821351 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -500,33 +500,32 @@ def __call__(self, **kwargs): # Ok, we have checked the input, let's "consume" it. port_content.add_consumer(self, port_name) - # Here we will store the results. - results = None - + # Create output tensors. if len(output_port_defs) == 1: + # Get port name and type. out_name = list(output_port_defs)[0] out_type = output_port_defs[out_name] + # Create a single returned tensor. results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) - # Bind the output ports. + # Bind the "default" output ports. self._app_state.active_graph.bind_outputs(output_port_defs, [results]) - else: - results_list = [] + # Create output tensors. + output_tensors = [] for out_name, out_type in output_port_defs.items(): - results_list.append(NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,)) + output_tensors.append(NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,)) - # Creating ad-hoc class for returning from module's forward pass. + # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' - field_names = list(output_port_defs) - result_type = collections.namedtuple(typename=output_class_name, field_names=field_names,) + result_type = namedtuple(typename=output_class_name, field_names=output_port_defs.keys()) - # Bind the output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, results_list) + # Create the returned tuple object. + results = result_type(*output_tensors) - # Tie tuple of output tensors with corresponding names. - results = result_type(*results_list) + # Bind the "default" output ports. + self._app_state.active_graph.bind_outputs(output_port_defs, output_tensors) # Return the results. return results diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index b90dc4050c3a..0bdb881ff49d 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -255,6 +255,22 @@ def type(self): """ return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) + + #def serialize(self): + # """ + # Serializes tensor to a dictionary (yaml structure). + # """ + # # serialize type. + # return 1 + + #@classmethod + #def deserialize(cls): + # """ + # Deserializes tensor from a dictionary (yaml structure). + # """ + # return 2 + + @property def producer_args(self): """ diff --git a/tests/system/test_pytorch_trainers.py b/tests/system/test_pytorch_trainers.py index 0c3b489b0888..3b8c9176fcfe 100644 --- a/tests/system/test_pytorch_trainers.py +++ b/tests/system/test_pytorch_trainers.py @@ -28,7 +28,7 @@ class TestPytorchTrainers(TestCase): @pytest.mark.system def test_simple_train(self): """ Simplest train test """ - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=1000, batch_size=128) trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) loss = nemo.backends.pytorch.tutorials.MSELoss() x, y = data_source() @@ -48,11 +48,6 @@ def test_simple_train_named_output(self): loss = nemo.backends.pytorch.tutorials.MSELoss() data = data_source() - self.assertEqual( - first=type(data).__name__, - second='RealFunctionDataLayerOutput', - msg='Check output class naming coherence.', - ) y_pred = trainable_module(x=data.x) loss_tensor = loss(predictions=y_pred, target=data.y) @@ -64,7 +59,7 @@ def test_simple_train_named_output(self): @pytest.mark.system def test_simple_chained_train(self): """ Chained train test """ - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=32) + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=1000, batch_size=32) trainable_module1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) trainable_module2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=2) trainable_module3 = nemo.backends.pytorch.tutorials.TaylorNet(dim=2) diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 3b54bb790b66..01d676f92dcb 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -49,13 +49,27 @@ def test_nm_tensors_producer_args(self): assert x.producer is data_source assert x.producer_args == {} + @pytest.mark.unit + def test_simple_train_named_output(self): + """ Test named output """ + data_source = RealFunctionDataLayer(n=10, batch_size=1,) + # Get data + data = data_source() + # Check output class naming coherence. + assert type(data).__name__ == 'RealFunctionDataLayerOutput' + + # Check types. + assert data.x.compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert data.y.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit def test_nm_tensors_producer_consumers(self): """ Tests whether nmTensors are correct - checking producers and consumers. """ # Create modules. - data_source = RealFunctionDataLayer(n=100, batch_size=1, name="source") + data_source = RealFunctionDataLayer(n=10, batch_size=1, name="source") trainable_module = TaylorNet(dim=4, name="tm") loss = MSELoss(name="loss") loss2 = MSELoss(name="loss2") @@ -103,7 +117,7 @@ def test_nm_tensors_types(self): Tests whether nmTensors are correct - checking type property. """ # Create modules. - data_source = RealFunctionDataLayer(n=100, batch_size=1) + data_source = RealFunctionDataLayer(n=10, batch_size=1) trainable_module = TaylorNet(dim=4) loss = MSELoss() From c9b2585f633b205735755dc39ddafb1387811e8c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 11:59:09 -0700 Subject: [PATCH 030/106] NmTensor test cleanup Signed-off-by: Tomasz Kornuta --- tests/system/test_pytorch_trainers.py | 24 ++++-------------------- tests/unit/core/test_nm_tensor.py | 1 + 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/system/test_pytorch_trainers.py b/tests/system/test_pytorch_trainers.py index 3b8c9176fcfe..f3f442a06444 100644 --- a/tests/system/test_pytorch_trainers.py +++ b/tests/system/test_pytorch_trainers.py @@ -28,7 +28,7 @@ class TestPytorchTrainers(TestCase): @pytest.mark.system def test_simple_train(self): """ Simplest train test """ - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=1000, batch_size=128) + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=100, batch_size=32) trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) loss = nemo.backends.pytorch.tutorials.MSELoss() x, y = data_source() @@ -37,29 +37,13 @@ def test_simple_train(self): optimizer = nemo.backends.pytorch.actions.PtActions() optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1}, - ) - - @pytest.mark.system - def test_simple_train_named_output(self): - """ Simplest train test with using named output """ - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=128,) - trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - loss = nemo.backends.pytorch.tutorials.MSELoss() - - data = data_source() - y_pred = trainable_module(x=data.x) - loss_tensor = loss(predictions=y_pred, target=data.y) - - optimizer = nemo.backends.pytorch.actions.PtActions() - optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1}, + tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 2}, ) @pytest.mark.system def test_simple_chained_train(self): """ Chained train test """ - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=1000, batch_size=32) + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=100, batch_size=32) trainable_module1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) trainable_module2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=2) trainable_module3 = nemo.backends.pytorch.tutorials.TaylorNet(dim=2) @@ -72,5 +56,5 @@ def test_simple_chained_train(self): optimizer = nemo.backends.pytorch.actions.PtActions() optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1}, + tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 2}, ) diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 01d676f92dcb..5f9013d30401 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -55,6 +55,7 @@ def test_simple_train_named_output(self): data_source = RealFunctionDataLayer(n=10, batch_size=1,) # Get data data = data_source() + # Check output class naming coherence. assert type(data).__name__ == 'RealFunctionDataLayerOutput' From adeecccbcceeac106f1446d4a1ec41dac57201c4 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 11:59:59 -0700 Subject: [PATCH 031/106] style fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_types/neural_type.py | 8 +++----- tests/unit/core/test_nm_tensor.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 0bdb881ff49d..d398e9f564f3 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -255,22 +255,20 @@ def type(self): """ return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) - - #def serialize(self): + # def serialize(self): # """ # Serializes tensor to a dictionary (yaml structure). # """ # # serialize type. # return 1 - #@classmethod - #def deserialize(cls): + # @classmethod + # def deserialize(cls): # """ # Deserializes tensor from a dictionary (yaml structure). # """ # return 2 - @property def producer_args(self): """ diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 5f9013d30401..3b0a68b39c38 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -55,7 +55,7 @@ def test_simple_train_named_output(self): data_source = RealFunctionDataLayer(n=10, batch_size=1,) # Get data data = data_source() - + # Check output class naming coherence. assert type(data).__name__ == 'RealFunctionDataLayerOutput' @@ -63,7 +63,6 @@ def test_simple_train_named_output(self): assert data.x.compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME assert data.y.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME - @pytest.mark.unit def test_nm_tensors_producer_consumers(self): """ From 127307b8e0967d1578197e4793ab7542b5d3fb66 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 12:31:48 -0700 Subject: [PATCH 032/106] LGTM cleanups Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests1.py | 23 +++++++++---------- .../graph_composition_integration_tests2.py | 19 ++++++++------- .../graph_composition_integration_tests3.py | 19 ++++++++------- .../graph_composition_integration_tests4.py | 21 ++++++++--------- examples/start_here/module_configuration.py | 15 ++++++------ .../start_here/module_custom_configuration.py | 12 ++++------ nemo/core/neural_graph.py | 14 +++++------ nemo/core/neural_modules.py | 1 - tests/unit/core/test_neural_graphs.py | 4 ++-- 9 files changed, 59 insertions(+), 69 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index e46028aa9c57..429a32bac5b6 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -17,16 +17,15 @@ # limitations under the License. # ============================================================================= -import nemo -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralModuleFactory, DeviceType, NeuralGraph, OperationMode, SimpleLossLoggerCallback +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.utils import logging -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() +nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -m2 = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32) +m2 = TaylorNet(dim=4) +loss = MSELoss() logging.info("This example shows how one can build an `explicit` graph.") @@ -37,13 +36,13 @@ # Manual bind. g0.output_ports["output"] = loss -print(g0.output_ports) -print(g0.output_ports["x"]) +#print(g0.output_ports) +#print(g0.output_ports["x"]) # SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( +callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), ) # Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py index d4de7eb72c05..5a24ce792938 100644 --- a/examples/start_here/graph_composition_integration_tests2.py +++ b/examples/start_here/graph_composition_integration_tests2.py @@ -17,16 +17,15 @@ # limitations under the License. # ============================================================================= -import nemo -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.utils import logging -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() +nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -m2 = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32) +m2 = TaylorNet(dim=4) +loss = MSELoss() logging.info( "This example shows how one can nest one graph into another - with binding of output ports." @@ -43,9 +42,9 @@ lss = loss(predictions=p1, target=t1) # SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( +callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), ) # Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests3.py b/examples/start_here/graph_composition_integration_tests3.py index 5aa3e0aead16..75d189b326c6 100644 --- a/examples/start_here/graph_composition_integration_tests3.py +++ b/examples/start_here/graph_composition_integration_tests3.py @@ -17,16 +17,15 @@ # limitations under the License. # ============================================================================= -import nemo -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.utils import logging -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() +nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -fx = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32) +fx = TaylorNet(dim=4) +loss = MSELoss() logging.info( "This example shows how one can nest one graph into another - with binding of the input ports." @@ -48,9 +47,9 @@ lss = loss(predictions=p, target=t) # SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( +callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), ) # Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd") +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests4.py b/examples/start_here/graph_composition_integration_tests4.py index e4c499143fd3..112cbbfa0ab8 100644 --- a/examples/start_here/graph_composition_integration_tests4.py +++ b/examples/start_here/graph_composition_integration_tests4.py @@ -19,17 +19,16 @@ import torch -import nemo -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback, EvaluatorCallback +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.utils import logging -logging = nemo.logging - -nf = nemo.core.NeuralModuleFactory() +nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl_training = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -dl_validation = nemo.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) -fx = nemo.tutorials.TaylorNet(dim=4) -loss = nemo.tutorials.MSELoss() +dl_training = RealFunctionDataLayer(n=100, batch_size=32) +dl_validation = RealFunctionDataLayer(n=100, batch_size=32) +fx = TaylorNet(dim=4) +loss = MSELoss() logging.info( "This example shows how one can nest one graph (representing the our trained model) into" @@ -60,7 +59,7 @@ # Callbacks to print info to console and Tensorboard. -train_callback = nemo.core.SimpleLossLoggerCallback( +train_callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') ) @@ -79,7 +78,7 @@ def batch_loss_epoch_finished_callback(global_vars): return dict({"Evaluation Loss": epoch_loss}) -eval_callback = nemo.core.EvaluatorCallback( +eval_callback = EvaluatorCallback( eval_tensors=[loss_e], user_iter_callback=batch_loss_per_batch_callback, user_epochs_done_callback=batch_loss_epoch_finished_callback, diff --git a/examples/start_here/module_configuration.py b/examples/start_here/module_configuration.py index 80f5a96e9635..fc1c65caa3a4 100644 --- a/examples/start_here/module_configuration.py +++ b/examples/start_here/module_configuration.py @@ -17,22 +17,21 @@ # limitations under the License. # ============================================================================= -import nemo -from nemo.core import DeviceType, NeuralModule, NeuralModuleFactory - -logging = nemo.logging +from nemo.core import DeviceType, NeuralModule, NeuralModuleFactory, SimpleLossLoggerCallback +from nemo.utils import logging +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet # Run on CPU. nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantitate RealFunctionDataLayer defaults to f=torch.sin, sampling from x=[-1, 1] -dl = nemo.tutorials.RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=128) +dl = RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=128) # Instantiate a simple feed-forward, single layer neural network. -fx = nemo.tutorials.TaylorNet(dim=4) +fx = TaylorNet(dim=4) # Instantitate loss. -mse_loss = nemo.tutorials.MSELoss() +mse_loss = MSELoss() # Export the model configuration. fx.export_to_config("/tmp/taylor_net.yml") @@ -47,7 +46,7 @@ loss = mse_loss(predictions=p, target=y) # SimpleLossLoggerCallback will print loss values to console. -callback = nemo.core.SimpleLossLoggerCallback( +callback = SimpleLossLoggerCallback( tensors=[loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') ) diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index 673a090f089e..8d95e9ac9514 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -22,11 +22,9 @@ from ruamel import yaml -import nemo from nemo.core import DeviceType, NeuralModuleFactory, SimpleLossLoggerCallback -from nemo.core.neural_types import ChannelType, NeuralType - -logging = nemo.logging +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.utils import logging # A custom enum. @@ -35,7 +33,7 @@ class Status(Enum): error = 1 -class CustomTaylorNet(nemo.tutorials.TaylorNet): +class CustomTaylorNet(TaylorNet): """Module which learns Taylor's coefficients. Extends the original module by a custom status enum.""" def __init__(self, dim, status: Status): @@ -123,13 +121,13 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantitate RealFunctionDataLayer defaults to f=torch.sin, sampling from x=[-1, 1] -dl = nemo.tutorials.RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=128) +dl = RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=32) # Instantiate a simple feed-forward, single layer neural network. fx = CustomTaylorNet(dim=4, status=Status.error) # Instantitate loss. -mse_loss = nemo.tutorials.MSELoss() +mse_loss = MSELoss() # Export the model configuration. fx.export_to_config("/tmp/custom_taylor_net.yml") diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index fa5c1ac39983..0d3189ea99f9 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -16,10 +16,9 @@ # limitations under the License. # ============================================================================= -import collections from typing import Dict, Optional +from collections import namedtuple -import nemo from nemo.core import OperationMode from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import ( @@ -97,7 +96,6 @@ def __call__(self, **kwargs): # print(" Neural Graph {} __call__".format(self._name)) # Get input and output ports definitions. input_port_defs = self.input_ports - output_port_defs = self.output_ports # TODO: check graph operation mode compatibility. @@ -114,7 +112,7 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) # Check what was actually passed. - if isinstance(port_content, nemo.core.NeuralGraph): + if isinstance(port_content, NeuralGraph): # TODO: make sure that port_content == self._app_state.active_graph ?!?! @@ -157,16 +155,16 @@ def __call__(self, **kwargs): # Create the module outputs. # This part is different from Neural Module. # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! - if len(output_port_defs) == 1: + if len(self._bound_output_tensors_default) == 1: # Return the single tensor. results = next(iter(self._bound_output_tensors_default.values())) else: # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' - result_type = namedtuple(typename=output_class_name, field_names=output_port_defs.keys()) + result_type = namedtuple(typename=output_class_name, field_names=self._bound_output_tensors_default.keys()) - # Bind the "default" output ports. - results = result_type(*self._bound_output_tensors_default) + # Return the "default" bound output ports. + results = result_type(*self._bound_output_tensors_default.values()) # Return the results. return results diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index c2db6d821351..746e24349340 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -18,7 +18,6 @@ """This file contains NeuralModule and NmTensor classes.""" __all__ = ['WeightShareTransform', 'NeuralModule'] -import collections import uuid from abc import abstractmethod from collections import namedtuple diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/test_neural_graphs.py index 003ec2941a63..dcb15f041f3e 100644 --- a/tests/unit/core/test_neural_graphs.py +++ b/tests/unit/core/test_neural_graphs.py @@ -21,7 +21,7 @@ import pytest from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import NeuralGraph, OperationMode +from nemo.core import NeuralGraph from nemo.core.neural_types import NeuralTypeComparisonResult @@ -89,7 +89,7 @@ def test_default_output_ports(self): m2 = TaylorNet(dim=4) loss = MSELoss() - with NeuralGraph(operation_mode=OperationMode.both) as g1: + with NeuralGraph() as g1: x, t = dl() p = m2(x=x) From 7b52fbeede6671d91b95e386e29ff506159f38b2 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 12:32:18 -0700 Subject: [PATCH 033/106] style fixes Signed-off-by: Tomasz Kornuta --- .../start_here/graph_composition_integration_tests1.py | 6 +++--- .../start_here/graph_composition_integration_tests2.py | 2 +- .../start_here/graph_composition_integration_tests3.py | 2 +- .../start_here/graph_composition_integration_tests4.py | 9 ++++++++- examples/start_here/module_configuration.py | 2 +- examples/start_here/module_custom_configuration.py | 2 +- nemo/core/neural_graph.py | 4 ++-- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index 429a32bac5b6..1bf11a732ab8 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -17,8 +17,8 @@ # limitations under the License. # ============================================================================= -from nemo.core import NeuralModuleFactory, DeviceType, NeuralGraph, OperationMode, SimpleLossLoggerCallback from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging nf = NeuralModuleFactory(placement=DeviceType.CPU) @@ -36,8 +36,8 @@ # Manual bind. g0.output_ports["output"] = loss -#print(g0.output_ports) -#print(g0.output_ports["x"]) +# print(g0.output_ports) +# print(g0.output_ports["x"]) # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py index 5a24ce792938..ad0b234faf9c 100644 --- a/examples/start_here/graph_composition_integration_tests2.py +++ b/examples/start_here/graph_composition_integration_tests2.py @@ -17,8 +17,8 @@ # limitations under the License. # ============================================================================= -from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging nf = NeuralModuleFactory(placement=DeviceType.CPU) diff --git a/examples/start_here/graph_composition_integration_tests3.py b/examples/start_here/graph_composition_integration_tests3.py index 75d189b326c6..007a99a06fe9 100644 --- a/examples/start_here/graph_composition_integration_tests3.py +++ b/examples/start_here/graph_composition_integration_tests3.py @@ -17,8 +17,8 @@ # limitations under the License. # ============================================================================= -from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging nf = NeuralModuleFactory(placement=DeviceType.CPU) diff --git a/examples/start_here/graph_composition_integration_tests4.py b/examples/start_here/graph_composition_integration_tests4.py index 112cbbfa0ab8..eaf3707a694e 100644 --- a/examples/start_here/graph_composition_integration_tests4.py +++ b/examples/start_here/graph_composition_integration_tests4.py @@ -19,8 +19,15 @@ import torch -from nemo.core import NeuralGraph, DeviceType, OperationMode, NeuralModuleFactory, SimpleLossLoggerCallback, EvaluatorCallback from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import ( + DeviceType, + EvaluatorCallback, + NeuralGraph, + NeuralModuleFactory, + OperationMode, + SimpleLossLoggerCallback, +) from nemo.utils import logging nf = NeuralModuleFactory(placement=DeviceType.CPU) diff --git a/examples/start_here/module_configuration.py b/examples/start_here/module_configuration.py index fc1c65caa3a4..91dbca88d830 100644 --- a/examples/start_here/module_configuration.py +++ b/examples/start_here/module_configuration.py @@ -17,9 +17,9 @@ # limitations under the License. # ============================================================================= +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralModule, NeuralModuleFactory, SimpleLossLoggerCallback from nemo.utils import logging -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet # Run on CPU. nf = NeuralModuleFactory(placement=DeviceType.CPU) diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index 8d95e9ac9514..4b66a84480f8 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -22,8 +22,8 @@ from ruamel import yaml -from nemo.core import DeviceType, NeuralModuleFactory, SimpleLossLoggerCallback from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralModuleFactory, SimpleLossLoggerCallback from nemo.utils import logging diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 0d3189ea99f9..d80d7272d0d9 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -16,8 +16,8 @@ # limitations under the License. # ============================================================================= -from typing import Dict, Optional from collections import namedtuple +from typing import Dict, Optional from nemo.core import OperationMode from nemo.core.neural_interface import NeuralInterface @@ -112,7 +112,7 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) # Check what was actually passed. - if isinstance(port_content, NeuralGraph): + if isinstance(port_content, NeuralGraph): # TODO: make sure that port_content == self._app_state.active_graph ?!?! From 81b87ea348531cddd4d14bac3acc3733566d95ea Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 13:54:13 -0700 Subject: [PATCH 034/106] BoundOutputs class and unit tests Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 2 +- nemo/utils/__init__.py | 1 + nemo/utils/bound_outputs.py | 93 ++++++++++++++++++++++++++ tests/unit/utils/test_bound_outputs.py | 73 ++++++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 nemo/utils/bound_outputs.py create mode 100644 tests/unit/utils/test_bound_outputs.py diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index d80d7272d0d9..b1a6e9b0da53 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -112,7 +112,7 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) # Check what was actually passed. - if isinstance(port_content, NeuralGraph): + if isinstance(port_content, NeuralGraph): # TODO: make sure that port_content == self._app_state.active_graph ?!?! diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index d67c8ee31b77..96e808d91ddb 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -25,3 +25,4 @@ from .helpers import * from nemo.utils.app_state import AppState from nemo.utils.object_registry import ObjectRegistry +from nemo.utils.bound_outputs import BoundOutputs diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py new file mode 100644 index 000000000000..b1e0092f037f --- /dev/null +++ b/nemo/utils/bound_outputs.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from collections.abc import MutableMapping + +from nemo.utils import logging + + +class BoundOutputs(MutableMapping): + ''' + A specialized dictionary that contains bound outputs. + In fact stores two lists of bound tensors ("default" and "manual"), and accesses them following the logic: + 1) Record all output tensors as "default" + 2) If user doesn't set any outputs manually, use the default. + ''' + + def __init__(self): + """ Initializes two dictionaries. """ + self._default_dict = {} + self._manual_dict = {} + + def __setitem__(self, key, value): + """ This method is used to set the manual dictionary. """ + if key in self._manual_dict.keys(): + raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) + # Ok, set output. + self._manual_dict[key] = value + + def __getitem__(self, key): + """ Returns output - depending whether there are some manual outputs or not. """ + if len(self._manual_dict) > 0: + return self._manual_dict[key] + else: # Use default dict. + return self._default_dict[key] + + def __delitem__(self, key): + raise NotImplemented("Deleting a bound output port is not allowed") + + def __iter__(self): + """ Iterates over the outputs - depending whether there are some manual outputs or not. """ + if len(self._manual_dict) > 0: + return iter(self._manual_dict) + else: # Use default dict. + return iter(self._default_dict) + + def __len__(self): + """ Return number of outputs - depending whether there are some manual outputs or not. """ + if len(self._manual_dict) > 0: + return len(self._manual_dict) + else: # Use default dict. + return len(self._default_dict) + + def add_defaults(self, tensors_list): + """ Binds default output tensors. + + Args: + tensors_list: List of tensors to be added. + """ + for tensor in tensors_list: + # Check the presence of the port name in default dictionary. + name = tensor.name # Use the default port name. + if name in self._default_dict.keys(): + logging.warning( + "Overwriting the already bound output port `{}` produced by `{}`".format( + name, self._default_dict[name].producer.name + ) + ) + # Still, overwrite it. + self._default_dict[name] = tensor + + @property + def definitions(self): + """ Property returns definitions of the output ports by extracting them on the fly from the bound tensors. """ + # Get dict. + d = self._manual_dict if len(self._manual_dict) > 0 else self._default_dict + + # Extract port definitions (Neural Types) straight from tensors. + return {k: v.type for k, v in d.items()} diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py new file mode 100644 index 000000000000..a1b81124ae5f --- /dev/null +++ b/tests/unit/utils/test_bound_outputs.py @@ -0,0 +1,73 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest + +# from nemo.core.neural_types import ( +# NeuralType, +# NeuralTypeComparisonResult, +# LossType, +# AudioSignal +# ) +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core.neural_types import NeuralTypeComparisonResult +from nemo.utils.bound_outputs import BoundOutputs + + +@pytest.mark.usefixtures("neural_factory") +class TestBoundOutputs: + @pytest.mark.unit + def test_binding(self): + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1) + tn = TaylorNet(dim=4) + loss = MSELoss() + + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = tn(x=x) + lss = loss(predictions=y_pred, target=y) + + # Test default binding. + bound_outputs = BoundOutputs() + + bound_outputs.add_defaults([x, y]) + bound_outputs.add_defaults([y_pred]) + bound_outputs.add_defaults([lss]) + + assert len(bound_outputs) == 4 + defs = bound_outputs.definitions + assert defs["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert defs["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert defs["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert defs["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + with pytest.raises(KeyError): + _ = defs["lss"] + + # And now bound manually. + bound_outputs["my_prediction"] = y_pred + bound_outputs["my_loss"] = lss + + assert len(bound_outputs) == 2 + defs = bound_outputs.definitions + assert defs["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert defs["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + with pytest.raises(KeyError): + _ = defs["x"] From fbfc87e476a756ecab200379f68d7814f26e8e77 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 13:54:21 -0700 Subject: [PATCH 035/106] style fixes Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 2 +- nemo/utils/bound_outputs.py | 2 +- tests/unit/utils/test_bound_outputs.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index b1a6e9b0da53..d80d7272d0d9 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -112,7 +112,7 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) # Check what was actually passed. - if isinstance(port_content, NeuralGraph): + if isinstance(port_content, NeuralGraph): # TODO: make sure that port_content == self._app_state.active_graph ?!?! diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index b1e0092f037f..68d4ed0060e8 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -73,7 +73,7 @@ def add_defaults(self, tensors_list): """ for tensor in tensors_list: # Check the presence of the port name in default dictionary. - name = tensor.name # Use the default port name. + name = tensor.name # Use the default port name. if name in self._default_dict.keys(): logging.warning( "Overwriting the already bound output port `{}` produced by `{}`".format( diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index a1b81124ae5f..a45ed4e0b98e 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -56,7 +56,7 @@ def test_binding(self): assert defs["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME assert defs["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert defs["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - + with pytest.raises(KeyError): _ = defs["lss"] From 41ee935c263ae610776aec75009e1abc61258f9b Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 16:43:39 -0700 Subject: [PATCH 036/106] manual output port binding operational Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 105 +++++++++--------- nemo/core/neural_modules.py | 4 +- nemo/utils/bound_outputs.py | 4 +- .../core/test_neural_graph_nesting.py | 2 +- tests/unit/core/test_neural_graph_nesting.py | 31 ++++++ tests/unit/core/test_nm_tensor.py | 2 +- tests/unit/utils/test_bound_outputs.py | 21 +++- 7 files changed, 104 insertions(+), 65 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index d80d7272d0d9..2d7f9eb7c682 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -28,6 +28,8 @@ NeuralTypeComparisonResult, ) +from nemo.utils.bound_outputs import BoundOutputs + class NeuralGraph(NeuralInterface): """ @@ -59,12 +61,8 @@ def __init__(self, operation_mode=OperationMode.both, name=None): # input port will be connected. self._bound_input_modules = {} - # Output ports and tensors - used in manual binding, empty for now. - self._bound_output_tensors_manual = {} - self._bound_output_ports_manual = {} - # Default output ports and tensors - used in automatic binding, empty for now. - self._bound_output_tensors_default = {} - self._bound_output_ports_default = {} + # Bound outputs. + self._bound_outputs = BoundOutputs() # "Modules" - list of modules constituting edges in a given graph. self._modules = {} @@ -105,26 +103,26 @@ def __call__(self, **kwargs): # print(self._steps) - # Iterate through all passed parameters. - for port_name, port_content in kwargs.items(): + # Iterate through all passed parameters - input ports. + # Port content: NmTensor or NeuralGraph (binding). + for input_port_name, input_object in kwargs.items(): # make sure that passed arguments correspond to input port names - if port_name not in input_port_defs.keys(): - raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) + if input_port_name not in input_port_defs.keys(): + raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(input_port_name)) # Check what was actually passed. - if isinstance(port_content, NeuralGraph): + if isinstance(input_object, NeuralGraph): - # TODO: make sure that port_content == self._app_state.active_graph ?!?! + # TODO: make sure that input_object == self._app_state.active_graph ?!?! # Bind this input port to a neural graph. - port_content.bind_input(port_name, input_port_defs[port_name], self) + input_object.bind_input(input_port_name, input_port_defs[input_port_name], self) # It is "compatible by definition";), so we don't have to check this port further. - else: # : port_content is a neural module. + else: # : input_object is a Tensor! # Compare input port definition with the received definition. - input_port = input_port_defs[port_name] - type_comatibility = input_port.compare(port_content) + type_comatibility = input_port_defs[input_port_name].compare(input_object) if ( type_comatibility != NeuralTypeComparisonResult.SAME and type_comatibility != NeuralTypeComparisonResult.GREATER @@ -135,36 +133,42 @@ def __call__(self, **kwargs): "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" "\n\nType comparison result: {4}".format( self.__class__.__name__, - port_name, - input_port_defs[port_name], - port_content, + input_port_name, + input_port_defs[input_port_name], + input_object, type_comatibility, ) ) + # Reaching that point means that we accepted input to a bound port. + # Need to connect it - add bound module as consumer. + consumer = self._bound_input_modules[input_port_name] + port_name = input_port_name # For now! + input_object.add_consumer(consumer, port_name) + # The current graph parsing requires us to update all outputs of # a module that "accepted" the input. - # Update means changing the original producer_args for the bound port. - producer = self._bound_input_modules[port_name] - for _, output_tensor in self._bound_output_tensors_default.items(): - if output_tensor.producer == producer: + # Update means changing the original producer_args for ALL IN THE GRAPH!! # TODO! + producer = self._bound_input_modules[input_port_name] + for output_tensor in self._bound_outputs.values(): + if output_tensor.producer.name == producer.name: # Set "input port value" to new content - which indicates tensor (and producer) # that will be used during graph backward traverse. - output_tensor.producer_args[port_name] = port_content + output_tensor.producer_args[port_name] = input_object # i.e. Tensor. # Create the module outputs. # This part is different from Neural Module. # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! - if len(self._bound_output_tensors_default) == 1: + if len(self._bound_outputs) == 1: # Return the single tensor. - results = next(iter(self._bound_output_tensors_default.values())) + results = next(iter(self._bound_outputs.values())) else: # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' - result_type = namedtuple(typename=output_class_name, field_names=self._bound_output_tensors_default.keys()) + result_type = namedtuple(typename=output_class_name, field_names=self._bound_outputs.keys()) # Return the "default" bound output ports. - results = result_type(*self._bound_output_tensors_default.values()) + results = result_type(*self._bound_outputs.values()) # Return the results. return results @@ -179,25 +183,27 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: return self._bound_input_ports @property - def output_ports(self) -> Optional[Dict[str, NeuralType]]: - """Returns definitions of module output ports + def output_ports(self): + """ + Returns module output ports. + + .. note:: + This method is NOT returning the dictionary with definitions (like Neural Module), + but the OutputPorts object. This was required to enable user to override the "default bound outputs" + with classical __setitem__ statement. + Returns: - A (dict) of module's output ports names to NeuralTypes mapping + A module output ports object. + """ - # print("getter!") - return self._bound_output_ports_default + return self._bound_outputs @property def operation_mode(self): """ Returns operation mode. """ return self._operation_mode - # @output_ports.setter - # def output_ports(self, ports): - # print("setter!") - # self._bound_output_ports_default = ports - def __enter__(self): """ Activates this graph. @@ -276,22 +282,13 @@ def bind_input(self, port_name, port_definition, bound_module): # Additionally, remember the bound module self._bound_input_modules[port_name] = bound_module - def bind_outputs(self, output_port_defs, output_values): - # print("Binding ALL outputs: defs = `{}`, values = `{}`".format(output_port_defs, output_values)) - - for (output_name, output_definition), output_value in zip(output_port_defs.items(), output_values): - # print( - # "Binding output: key = `{}`: def = `{}`, value = `{}`".format( - # output_name, type(output_definition), type(output_value) - # ) - # ) - # Copy the definition of the port to graph input port definition. - self._bound_output_ports_default[output_name] = output_definition - - # Bind output tensors. - self._bound_output_tensors_default[output_name] = output_value - # Additionally, store all output tensors. - # self._all_output_tensors[output_name] = output_value + def bind_default_outputs(self, tensors_list): + """ Binds default outputs. + + Args: + tensors_list: List of tensors to be added. + """ + self._bound_outputs.bind_defaults(tensors_list) def show_bound_inputs(self): print("bound input ports: ") diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 746e24349340..962e1f1a820e 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -509,7 +509,7 @@ def __call__(self, **kwargs): results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) # Bind the "default" output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, [results]) + self._app_state.active_graph.bind_default_outputs([results]) else: # Create output tensors. output_tensors = [] @@ -524,7 +524,7 @@ def __call__(self, **kwargs): results = result_type(*output_tensors) # Bind the "default" output ports. - self._app_state.active_graph.bind_outputs(output_port_defs, output_tensors) + self._app_state.active_graph.bind_default_outputs(output_tensors) # Return the results. return results diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index 68d4ed0060e8..1697e7272a41 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -49,7 +49,7 @@ def __getitem__(self, key): return self._default_dict[key] def __delitem__(self, key): - raise NotImplemented("Deleting a bound output port is not allowed") + raise NotImplementedError("Deleting a bound output port is not allowed") def __iter__(self): """ Iterates over the outputs - depending whether there are some manual outputs or not. """ @@ -65,7 +65,7 @@ def __len__(self): else: # Use default dict. return len(self._default_dict) - def add_defaults(self, tensors_list): + def bind_defaults(self, tensors_list): """ Binds default output tensors. Args: diff --git a/tests/integration/core/test_neural_graph_nesting.py b/tests/integration/core/test_neural_graph_nesting.py index 5b9b4e85a6a2..e5a7458da311 100644 --- a/tests/integration/core/test_neural_graph_nesting.py +++ b/tests/integration/core/test_neural_graph_nesting.py @@ -29,7 +29,7 @@ @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphNesting: @pytest.mark.integration - def test_nesting_operation_modes_ok(self): + def test_nesting_operation_modes_example(self): """ Tests whether one can nest one graph in mode `both` (representing the our `model`) into `training` and validation (`inference`) graphs. diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 8b9c39a27a01..aee55f95cc08 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -23,6 +23,7 @@ from nemo.backends.pytorch.actions import PtActions from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode, SimpleLossLoggerCallback +from nemo.core.neural_types import NeuralTypeComparisonResult from nemo.utils import logging @@ -106,3 +107,33 @@ def test_graph_nesting_possible_operation_modes(self): with pytest.raises(TypeError): with NeuralGraph(operation_mode=OperationMode.both): _, _ = inference() + + + def test_output_ports_binding(self): + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1) + tn = TaylorNet(dim=4) + loss = MSELoss() + + # Test default binding. + with NeuralGraph(operation_mode=OperationMode.training) as g1: + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = tn(x=x) + lss = loss(predictions=y_pred, target=y) + + assert len(g1.output_ports) == 4 + assert g1.output_ports["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + # Test manual binding. + with g1: + g1.output_ports["my_prediction"] = y_pred + g1.output_ports["my_loss"] = lss + + assert len(g1.output_ports) == 2 + assert g1.output_ports["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 3b0a68b39c38..1fd590a781cd 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -52,7 +52,7 @@ def test_nm_tensors_producer_args(self): @pytest.mark.unit def test_simple_train_named_output(self): """ Test named output """ - data_source = RealFunctionDataLayer(n=10, batch_size=1,) + data_source = RealFunctionDataLayer(n=10, batch_size=1) # Get data data = data_source() diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index a45ed4e0b98e..9abf44a05bf2 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -26,8 +26,8 @@ # ) from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core.neural_types import NeuralTypeComparisonResult -from nemo.utils.bound_outputs import BoundOutputs +from nemo.utils.bound_outputs import BoundOutputs @pytest.mark.usefixtures("neural_factory") class TestBoundOutputs: @@ -46,11 +46,16 @@ def test_binding(self): # Test default binding. bound_outputs = BoundOutputs() - bound_outputs.add_defaults([x, y]) - bound_outputs.add_defaults([y_pred]) - bound_outputs.add_defaults([lss]) + bound_outputs.bind_defaults([x, y]) + bound_outputs.bind_defaults([y_pred]) + bound_outputs.bind_defaults([lss]) + + # Delete not allowed. + with pytest.raises(NotImplementedError): + del bound_outputs["loss"] assert len(bound_outputs) == 4 + defs = bound_outputs.definitions assert defs["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME assert defs["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME @@ -60,10 +65,14 @@ def test_binding(self): with pytest.raises(KeyError): _ = defs["lss"] - # And now bound manually. + # Bound manually. bound_outputs["my_prediction"] = y_pred bound_outputs["my_loss"] = lss + # Delete not allowed. + with pytest.raises(NotImplementedError): + del bound_outputs["my_prediction"] + assert len(bound_outputs) == 2 defs = bound_outputs.definitions assert defs["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME @@ -71,3 +80,5 @@ def test_binding(self): with pytest.raises(KeyError): _ = defs["x"] + + From bce7e904c7b3ccc8fe38ad95988aebf8965f681e Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 10 Apr 2020 16:43:53 -0700 Subject: [PATCH 037/106] manual output port binding operational Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 11 +++++------ tests/unit/core/test_neural_graph_nesting.py | 2 -- tests/unit/utils/test_bound_outputs.py | 4 +--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 2d7f9eb7c682..e2a679b8443b 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -27,7 +27,6 @@ NeuralType, NeuralTypeComparisonResult, ) - from nemo.utils.bound_outputs import BoundOutputs @@ -103,8 +102,8 @@ def __call__(self, **kwargs): # print(self._steps) - # Iterate through all passed parameters - input ports. - # Port content: NmTensor or NeuralGraph (binding). + # Iterate through all passed parameters - input ports. + # Port content: NmTensor or NeuralGraph (binding). for input_port_name, input_object in kwargs.items(): # make sure that passed arguments correspond to input port names if input_port_name not in input_port_defs.keys(): @@ -139,11 +138,11 @@ def __call__(self, **kwargs): type_comatibility, ) ) - + # Reaching that point means that we accepted input to a bound port. # Need to connect it - add bound module as consumer. consumer = self._bound_input_modules[input_port_name] - port_name = input_port_name # For now! + port_name = input_port_name # For now! input_object.add_consumer(consumer, port_name) # The current graph parsing requires us to update all outputs of @@ -154,7 +153,7 @@ def __call__(self, **kwargs): if output_tensor.producer.name == producer.name: # Set "input port value" to new content - which indicates tensor (and producer) # that will be used during graph backward traverse. - output_tensor.producer_args[port_name] = input_object # i.e. Tensor. + output_tensor.producer_args[port_name] = input_object # i.e. Tensor. # Create the module outputs. # This part is different from Neural Module. diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index aee55f95cc08..9d6abadfa28f 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -108,7 +108,6 @@ def test_graph_nesting_possible_operation_modes(self): with NeuralGraph(operation_mode=OperationMode.both): _, _ = inference() - def test_output_ports_binding(self): # Create modules. data_source = RealFunctionDataLayer(n=100, batch_size=1) @@ -136,4 +135,3 @@ def test_output_ports_binding(self): assert len(g1.output_ports) == 2 assert g1.output_ports["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert g1.output_ports["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index 9abf44a05bf2..c2682ba99c80 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -26,9 +26,9 @@ # ) from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core.neural_types import NeuralTypeComparisonResult - from nemo.utils.bound_outputs import BoundOutputs + @pytest.mark.usefixtures("neural_factory") class TestBoundOutputs: @pytest.mark.unit @@ -80,5 +80,3 @@ def test_binding(self): with pytest.raises(KeyError): _ = defs["x"] - - From 752b81002f92b62f1c9a1482131738a5e3eca46c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 13 Apr 2020 19:26:35 -0700 Subject: [PATCH 038/106] Recording and updating all modules during connecting to a graph Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 9 ++-- nemo/core/neural_types/neural_type.py | 8 ++++ nemo/utils/bound_outputs.py | 61 ++++++++++++++++++-------- tests/unit/core/test_nm_tensor.py | 8 ++-- tests/unit/utils/test_bound_outputs.py | 6 --- 5 files changed, 59 insertions(+), 33 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index e2a679b8443b..6c7c2119c127 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -147,10 +147,11 @@ def __call__(self, **kwargs): # The current graph parsing requires us to update all outputs of # a module that "accepted" the input. - # Update means changing the original producer_args for ALL IN THE GRAPH!! # TODO! - producer = self._bound_input_modules[input_port_name] - for output_tensor in self._bound_outputs.values(): - if output_tensor.producer.name == producer.name: + # Update means changing the original producer_args for ALL (OUTPUT) TENSORS IN THE GRAPH!! + producer_name = self._bound_input_modules[input_port_name].name + if producer_name in self._bound_outputs.all.keys(): + # Get all tensor producer by this module. + for output_tensor in self._bound_outputs.all[producer_name]: # Set "input port value" to new content - which indicates tensor (and producer) # that will be used during graph backward traverse. output_tensor.producer_args[port_name] = input_object # i.e. Tensor. diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index d398e9f564f3..cf0aaf1d59e4 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -220,6 +220,14 @@ def producer(self): """ return self._producer + @property + def producer_name(self): + """ + Returns: + Name of the producer of the tensor. + """ + return self._producer.name + @property def producer_port(self): """ diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index 1697e7272a41..cd18f38af31e 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -31,39 +31,50 @@ class BoundOutputs(MutableMapping): def __init__(self): """ Initializes two dictionaries. """ - self._default_dict = {} - self._manual_dict = {} + # This dictionary stores list of output tensors of all modules, one key per + # This will generate a "all output tensors" dictionary, where the key is name of "producer" module, + # and the value contains all produced tensors. + self._all_tensors = {} + + # This dictionary stores the output tensors collected during the "default" tensor recording. + # As they are using the default port names, the second/next tensor published on the same port + # will overwrite the old one (Warning). + self._default_tensors = {} + + # This dictionary stores list of output tensors of module "manually" indicated by the user. + # In this case tring to overwriting the existing ports with new tensors will be forbidden (Exception). + self._manual_tensors = {} def __setitem__(self, key, value): """ This method is used to set the manual dictionary. """ - if key in self._manual_dict.keys(): + if key in self._manual_tensors.keys(): raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) # Ok, set output. - self._manual_dict[key] = value + self._manual_tensors[key] = value def __getitem__(self, key): """ Returns output - depending whether there are some manual outputs or not. """ - if len(self._manual_dict) > 0: - return self._manual_dict[key] + if len(self._manual_tensors) > 0: + return self._manual_tensors[key] else: # Use default dict. - return self._default_dict[key] + return self._default_tensors[key] def __delitem__(self, key): raise NotImplementedError("Deleting a bound output port is not allowed") def __iter__(self): """ Iterates over the outputs - depending whether there are some manual outputs or not. """ - if len(self._manual_dict) > 0: - return iter(self._manual_dict) + if len(self._manual_tensors) > 0: + return iter(self._manual_tensors) else: # Use default dict. - return iter(self._default_dict) + return iter(self._default_tensors) def __len__(self): """ Return number of outputs - depending whether there are some manual outputs or not. """ - if len(self._manual_dict) > 0: - return len(self._manual_dict) + if len(self._manual_tensors) > 0: + return len(self._manual_tensors) else: # Use default dict. - return len(self._default_dict) + return len(self._default_tensors) def bind_defaults(self, tensors_list): """ Binds default output tensors. @@ -72,22 +83,34 @@ def bind_defaults(self, tensors_list): tensors_list: List of tensors to be added. """ for tensor in tensors_list: - # Check the presence of the port name in default dictionary. + # Check the presence of the port name in "default" dictionary. name = tensor.name # Use the default port name. - if name in self._default_dict.keys(): + if name in self._default_tensors.keys(): logging.warning( "Overwriting the already bound output port `{}` produced by `{}`".format( - name, self._default_dict[name].producer.name + name, self._default_tensors[name].producer_name ) ) # Still, overwrite it. - self._default_dict[name] = tensor + self._default_tensors[name] = tensor + + # Add tensor to "all" tensors dictionary. + producer_name = tensor.producer_name + if producer_name not in self._all_tensors.keys(): + self._all_tensors[producer_name] = [] + # Add tensor. + self._all_tensors[producer_name].append(tensor) + + @property + def all(self): + """ Returns dictionary of dictionary of output tensors. """ + return self._all_tensors @property def definitions(self): """ Property returns definitions of the output ports by extracting them on the fly from the bound tensors. """ - # Get dict. - d = self._manual_dict if len(self._manual_dict) > 0 else self._default_dict + # Get the right tensor dictionary. + d = self._manual_tensors if len(self._manual_tensors) > 0 else self._default_tensors # Extract port definitions (Neural Types) straight from tensors. return {k: v.type for k, v in d.items()} diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 1fd590a781cd..bfe405e23214 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -40,13 +40,13 @@ def test_nm_tensors_producer_args(self): loss_tensor = loss(predictions=y_pred, target=y) # check producers' bookkeeping - assert loss_tensor.producer == loss + assert loss_tensor.producer_name == loss.name assert loss_tensor.producer_args == {"predictions": y_pred, "target": y} - assert y_pred.producer is trainable_module + assert y_pred.producer_name == trainable_module.name assert y_pred.producer_args == {"x": x} - assert y.producer is data_source + assert y.producer_name == data_source.name assert y.producer_args == {} - assert x.producer is data_source + assert x.producer_name == data_source.name assert x.producer_args == {} @pytest.mark.unit diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index c2682ba99c80..28063859af3f 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -18,12 +18,6 @@ import pytest -# from nemo.core.neural_types import ( -# NeuralType, -# NeuralTypeComparisonResult, -# LossType, -# AudioSignal -# ) from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core.neural_types import NeuralTypeComparisonResult from nemo.utils.bound_outputs import BoundOutputs From f68c8830db0524379277fd35c96ed909254d07a5 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 14 Apr 2020 20:39:52 -0700 Subject: [PATCH 039/106] work in progress on output port binding, not operations Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests2_1.py | 55 ++++++ .../graph_composition_integration_tests3_1.py | 59 ++++++ nemo/core/neural_graph.py | 183 ++++++++++-------- nemo/core/neural_modules.py | 97 ++++++---- nemo/core/neural_types/neural_type.py | 38 ++-- nemo/utils/__init__.py | 2 + nemo/utils/app_state.py | 7 +- nemo/utils/bound_inputs.py | 109 +++++++++++ nemo/utils/bound_outputs.py | 22 ++- nemo/utils/module_port.py | 27 +++ tests/unit/core/test_nm_tensor.py | 32 +-- 11 files changed, 466 insertions(+), 165 deletions(-) create mode 100644 examples/start_here/graph_composition_integration_tests2_1.py create mode 100644 examples/start_here/graph_composition_integration_tests3_1.py create mode 100644 nemo/utils/bound_inputs.py create mode 100644 nemo/utils/module_port.py diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py new file mode 100644 index 000000000000..172583114656 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -0,0 +1,55 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback +from nemo.utils import logging + +nf = NeuralModuleFactory(placement=DeviceType.CPU) +# Instantiate the necessary neural modules. +dl = RealFunctionDataLayer(n=100, batch_size=32) +m1 = TaylorNet(dim=4) +m2 = TaylorNet(dim=4) +loss = MSELoss() + +logging.info( + "This example shows how one can nest one graph into another - with manual binding of selected output ports." + F" Please note that the nested graph can be used exatly like any other module." +) + +with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: + x, t = dl() + prediction1 = m1(x=x) + prediction2 = m2(x=x) + # Manually bind the selected output ports. + g1.output_ports["x"] = x + g1.output_ports["t"] = t + g1.output_ports["prediction"] = prediction1 + +with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g11: + x1, t1, p1 = g1() + lss = loss(predictions=p1, target=t1) + +# SimpleLossLoggerCallback will print loss values to console. +callback = SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py new file mode 100644 index 000000000000..d8aa574652d7 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -0,0 +1,59 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback +from nemo.utils import logging + +nf = NeuralModuleFactory(placement=DeviceType.CPU) +# Instantiate the necessary neural modules. +dl = RealFunctionDataLayer(n=100, batch_size=32) +fx = TaylorNet(dim=4) +loss = MSELoss() + +logging.info( + "This example shows how one can nest one graph into another - with binding of the input ports." + F" Please note that the nested graph can be used exatly like any other module" + F" In particular, note that the input port 'x' of the module `m2` is bound in graph 'g2'" + F" and then set to `x` returned by `dl` in the graph `g3`." +) + +with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: + # Manually bind input port: "input" -> "x" + g2.input_ports["input"] = fx.input_ports["x"] + # Add module to graph and bind it input port 'x'. + y = fx(x=g2.input_ports["input"]) + # lss = loss(predictions=y, target=g2.input_ports["input"]) + +# Build the training graph. +with NeuralGraph(operation_mode=OperationMode.training, name="g3") as g3: + # Add modules to graph. + x, t = dl() + # Incorporate modules from the existing graph. + import pdb; pdb.set_trace() + p = g2(input=x) + lss = loss(predictions=p, target=t) + +# SimpleLossLoggerCallback will print loss values to console. +callback = SimpleLossLoggerCallback( + tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 6c7c2119c127..fb05285e2a41 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -23,10 +23,9 @@ from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import ( NeuralPortNameMismatchError, - NeuralPortNmTensorMismatchError, - NeuralType, - NeuralTypeComparisonResult, + NmTensor, ) +from nemo.utils.bound_inputs import BoundInput, BoundInputs from nemo.utils.bound_outputs import BoundOutputs @@ -60,12 +59,15 @@ def __init__(self, operation_mode=OperationMode.both, name=None): # input port will be connected. self._bound_input_modules = {} + # Bound inputs. + self._bound_inputs = BoundInputs() + # Bound outputs. self._bound_outputs = BoundOutputs() - # "Modules" - list of modules constituting edges in a given graph. + # "Modules" - list of modules constituting "nodes" in a given graph. self._modules = {} - # "Steps": ordered execution of modules in a graph. + # "Steps": order of the execution of modules in a graph. self._steps = [] def __call__(self, **kwargs): @@ -74,6 +76,8 @@ def __call__(self, **kwargs): Also checks if all inputs were provided and properly connects them. """ + # print(" Neural Graph {} __call__".format(self._name)) + # Test operation modes of the nested graphs. outer_mode = self._app_state.active_graph.operation_mode inner_mode = self.operation_mode @@ -90,73 +94,93 @@ def __call__(self, **kwargs): if inner_mode == OperationMode.inference and outer_mode == OperationMode.both: raise TypeError("Cannot nest 'inference' graph into 'both'") - # print(" Neural Graph {} __call__".format(self._name)) # Get input and output ports definitions. input_port_defs = self.input_ports - # TODO: check graph operation mode compatibility. - - # "Copy" all the operations from the previous graph. + # "Copy" all the operations from the previous graph. TODO better! for step in self._steps: self._app_state.active_graph.record_step(*step) - # print(self._steps) - # Iterate through all passed parameters - input ports. - # Port content: NmTensor or NeuralGraph (binding). - for input_port_name, input_object in kwargs.items(): + ###### PROCESS INPUTS. ###### + # Iterate through all passed parameters. + for port_name, port_content in kwargs.items(): # make sure that passed arguments correspond to input port names - if input_port_name not in input_port_defs.keys(): - raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(input_port_name)) + if port_name not in input_port_defs.keys(): + raise NeuralPortNameMismatchError(port_name) + + # Analogically to NeuralModule, at that point the input can be one of three types: + # * NeuralGraph -> bind port using the default name and type. + # * BoundInput -> check definition, if ok bind port + # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. - if isinstance(input_object, NeuralGraph): + if type(port_content) is NeuralGraph: + + # Make sure that port_content is the currently active graph! + if port_content is not self._app_state.active_graph: + raise ConnectionError("Ports can be bound only by passing the active graph object!") + + # This case: we are nesting one graph into another and must bind input port of one graph in another! + # So generally we will "copy" the BoundInput object. - # TODO: make sure that input_object == self._app_state.active_graph ?!?! + # Create an alias so the logic will be more clear. + active_graph = port_content - # Bind this input port to a neural graph. - input_object.bind_input(input_port_name, input_port_defs[input_port_name], self) + # Copy the port "definition" (i.e. is NeuralType) using the same port name. + # This might throw an exception if port with that name was already bound! + active_graph.input_ports[port_name] = input_port_defs[port_name].type - # It is "compatible by definition";), so we don't have to check this port further. + # Bind the neural graph input port, i.e. remember that a given graph port should pass data + # to all "bound modules" (when it finally will be connected). + active_graph.input_ports[port_name].bind(input_port_defs[port_name].modules) + + # Please note that there are no "consumers" here - this is a "pure" binding. + + elif type(port_content) is BoundInput: - else: # : input_object is a Tensor! # Compare input port definition with the received definition. - type_comatibility = input_port_defs[input_port_name].compare(input_object) - if ( - type_comatibility != NeuralTypeComparisonResult.SAME - and type_comatibility != NeuralTypeComparisonResult.GREATER - ): - raise NeuralPortNmTensorMismatchError( - "\n\nIn {0}. \n" - "Port: {1} and a NmTensor it was fed are \n" - "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" - "\n\nType comparison result: {4}".format( - self.__class__.__name__, - input_port_name, - input_port_defs[input_port_name], - input_object, - type_comatibility, - ) - ) - - # Reaching that point means that we accepted input to a bound port. - # Need to connect it - add bound module as consumer. - consumer = self._bound_input_modules[input_port_name] - port_name = input_port_name # For now! - input_object.add_consumer(consumer, port_name) - - # The current graph parsing requires us to update all outputs of - # a module that "accepted" the input. - # Update means changing the original producer_args for ALL (OUTPUT) TENSORS IN THE GRAPH!! - producer_name = self._bound_input_modules[input_port_name].name - if producer_name in self._bound_outputs.all.keys(): - # Get all tensor producer by this module. - for output_tensor in self._bound_outputs.all[producer_name]: - # Set "input port value" to new content - which indicates tensor (and producer) - # that will be used during graph backward traverse. - output_tensor.producer_args[port_name] = input_object # i.e. Tensor. - - # Create the module outputs. + input_port_defs[port_name].type.compare_and_raise_error( + self.__class__.__name__, port_name, port_content.type + ) + + # Bind the input port modules, i.e. remember that a given graph port should pass data + # to all "bound modules" (when it finally will be connected). + port_content.bind(self.modules) + + # Please note that there are no "consumers" here - this is a "pure" binding. + + elif type(port_content) is NmTensor: + # Compare input port definition with the received definition. + input_port_defs[port_name].type.compare_and_raise_error( + self.__class__.__name__, port_name, port_content + ) + + # Reaching that point means that we accepted input to a bound graph port. + # Need to connect it - "copy" all modules connected to this "bound input" as consumers. + for consumer in input_port_defs[port_name].modules: + # Add consumer. + port_content.add_consumer(consumer.module_name, consumer.port_name) + + # The current graph parsing requires us to update all outputs of + # a module that "accepted" the input. + # In other words: for every "consumer" we need to update all tensors it produced. + # Update means changing the original producer_args for ALL TENSORS IN THE GRAPH produced by + # this module. + producer_name = consumer.module_name + if producer_name in self._bound_outputs.all.keys(): + # Get all tensor producer by this module. + for output_tensor in self._bound_outputs.all[producer_name]: + # Set "input port value" to new content - which indicates tensor (and producer) + # that will be used during graph backward traverse. + output_tensor.producer_args[port_name] = port_content # i.e. Tensor. + + else: + raise TypeError( + "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) + ) + + ###### PRODUCE OUTPUTS. ###### # This part is different from Neural Module. # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! if len(self._bound_outputs) == 1: @@ -174,13 +198,18 @@ def __call__(self, **kwargs): return results @property - def input_ports(self) -> Optional[Dict[str, NeuralType]]: + def input_ports(self): """Returns definitions of module input ports. + .. note:: + This method is NOT returning the dictionary with definitions (like Neural Modules), + but the BoundInputs object. + This was required to enable user to bound inputs with the dict's __setitem__ construct. + Returns: - A (dict) of module's input ports names to NeuralTypes mapping + A graph bound input ports. """ - return self._bound_input_ports + return self._bound_inputs @property def output_ports(self): @@ -188,13 +217,13 @@ def output_ports(self): Returns module output ports. .. note:: - This method is NOT returning the dictionary with definitions (like Neural Module), - but the OutputPorts object. This was required to enable user to override the "default bound outputs" - with classical __setitem__ statement. - + This method is NOT returning the dictionary with definitions (like Neural Modules), + but the BoundOutputs object. + This was required to enable user to override the "default bound outputs" + with the dict's __setitem__ construct. Returns: - A module output ports object. + A graph bound output ports. """ return self._bound_outputs @@ -272,23 +301,23 @@ def record_step(self, module, inputs): # Add step. self._steps.append([module, inputs]) - def bind_input(self, port_name, port_definition, bound_module): - # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) - # Copy the definition of the port to graph input port definition. - self._bound_input_ports[port_name] = port_definition + # def bind_input(self, port_name, port_definition, bound_module): + # # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) + # # Copy the definition of the port to graph input port definition. + # self._bound_input_ports[port_name] = port_definition - # Indicate that this tensor is missing and has to be provided! - self._bound_input_tensors[port_name] = None - # Additionally, remember the bound module - self._bound_input_modules[port_name] = bound_module + # # Indicate that this tensor is missing and has to be provided! + # self._bound_input_tensors[port_name] = None + # # Additionally, remember the bound module + # self._bound_input_modules[port_name] = bound_module - def bind_default_outputs(self, tensors_list): - """ Binds default outputs. + def bind_outputs(self, tensors_list): + """ Binds the output tensors. Args: - tensors_list: List of tensors to be added. + tensors_list: List of tensors to be bound. """ - self._bound_outputs.bind_defaults(tensors_list) + self._bound_outputs.bind(tensors_list) def show_bound_inputs(self): print("bound input ports: ") diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 962e1f1a820e..ed51dc13ff18 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -30,16 +30,12 @@ from nemo.core import NeuralGraph, NeuralModuleFactory, OperationMode from nemo.core.neural_interface import NeuralInterface -from nemo.core.neural_types import ( - NeuralPortNameMismatchError, - NeuralPortNmTensorMismatchError, - NeuralType, - NeuralTypeComparisonResult, - NmTensor, -) +from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging +from nemo.utils.bound_inputs import BoundInput from nemo.utils.decorators.deprecated import deprecated +from nemo.utils.module_port import ModulePort YAML = YAML(typ='safe') @@ -449,11 +445,11 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ + # print(" Neural Module:__call__") # Set the operation mode of the outer graph. self.operation_mode = self._app_state.active_graph.operation_mode - # print(" Neural Module:__call__") # Get input and output ports definitions - potentially depending on the operation mode! input_port_defs = self.input_ports output_port_defs = self.output_ports @@ -461,44 +457,61 @@ def __call__(self, **kwargs): # Record the operation (i.e. add a single module). self._app_state.active_graph.record_step(self, kwargs.items()) + ###### PROCESS INPUTS. ###### # Iterate through all passed parameters. for port_name, port_content in kwargs.items(): - # make sure that passed arguments correspond to input port names + # Make sure that passed arguments corresponds to one of the input port names. if port_name not in input_port_defs.keys(): - raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) + raise NeuralPortNameMismatchError(port_name) + + # Ok, at that point the input can be one of three types: + # * NeuralGraph -> bind port using the default name and type. + # * BoundInput -> check definition, if ok bind port + # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. - if isinstance(port_content, NeuralGraph): - # Bind this input port to a neural graph. - - # TODO: make sure that port_content == self._app_state.active_graph ????? - if port_content != self._app_state.active_graph: - raise ConnectionError("Cannot bind ports of one graph with a different graph!") - port_content.bind_input(port_name, input_port_defs[port_name], self) - # It is "compatible by definition";), so we don't have to check this port further. - else: # : port_content is a neural module. + if type(port_content) is NeuralGraph: + # Make sure that port_content is the currently active graph! + if port_content is not self._app_state.active_graph: + raise ConnectionError("Ports can be bound only by passing the active graph object!") + # Create an alias so the logic will be more clear. + active_graph = port_content + + # Copy the port "definition" (i.e. is NeuralType) using the same port name. + active_graph.input_ports[port_name] = input_port_defs[port_name] + + # Bind the neural graph input port, i.e. remember that a given graph port should pass data + # to THIS module (when it finally will be connected). + active_graph.input_ports[port_name].bind([ModulePort(self.name, port_name)]) + + # Please note that there are no "consumers" here - this is a "pure" binding. + + elif type(port_content) is BoundInput: + # Compare input port definition with the received definition. - input_port = input_port_defs[port_name] - type_comatibility = input_port.compare(port_content) - if ( - type_comatibility != NeuralTypeComparisonResult.SAME - and type_comatibility != NeuralTypeComparisonResult.GREATER - ): - raise NeuralPortNmTensorMismatchError( - "\n\nIn {0}. \n" - "Port: {1} and a NmTensor it was fed are \n" - "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" - "\n\nType comparison result: {4}".format( - self.__class__.__name__, - port_name, - input_port_defs[port_name], - port_content, - type_comatibility, - ) - ) - # Ok, we have checked the input, let's "consume" it. - port_content.add_consumer(self, port_name) + input_port_defs[port_name].compare_and_raise_error( + self.__class__.__name__, port_name, port_content.type + ) + + # Bind the neural graph input port, i.e. remember that a given graph port should pass data + # to THIS module (when it finally will be connected). + port_content.bind([ModulePort(self.name, port_name)]) + + # Please note that there are no "consumers" here - this is a "pure" binding. + + elif type(port_content) is NmTensor: + # Compare input port definition with the received definition. + input_port_defs[port_name].compare_and_raise_error(self.__class__.__name__, port_name, port_content) + # Ok, can connect, add self (module) as "consumer" to the input tensor. + port_content.add_consumer(self.name, port_name) + + else: + raise TypeError( + "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) + ) + + ###### PRODUCE OUTPUTS. ###### # Create output tensors. if len(output_port_defs) == 1: # Get port name and type. @@ -509,7 +522,7 @@ def __call__(self, **kwargs): results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) # Bind the "default" output ports. - self._app_state.active_graph.bind_default_outputs([results]) + self._app_state.active_graph.bind_outputs([results]) else: # Create output tensors. output_tensors = [] @@ -523,8 +536,8 @@ def __call__(self, **kwargs): # Create the returned tuple object. results = result_type(*output_tensors) - # Bind the "default" output ports. - self._app_state.active_graph.bind_default_outputs(output_tensors) + # Bind the output tensors. + self._app_state.active_graph.bind_outputs(output_tensors) # Return the results. return results diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index cf0aaf1d59e4..445c17d02710 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -21,15 +21,14 @@ 'NeuralTypeError', 'NeuralPortNameMismatchError', 'NeuralPortNmTensorMismatchError', - 'CanNotInferResultNeuralType', ] import uuid -from collections import namedtuple from typing import Optional, Tuple from nemo.core.neural_types.axes import AxisKind, AxisType from nemo.core.neural_types.comparison import NeuralTypeComparisonResult from nemo.core.neural_types.elements import * +from nemo.utils.module_port import ModulePort class NeuralType(object): @@ -113,6 +112,15 @@ def compare(self, second) -> NeuralTypeComparisonResult: else: return NeuralTypeComparisonResult.INCOMPATIBLE + def compare_and_raise_error(self, parent_type_name, port_name, second_object): + """ Method compares definition of one type with another and raises an error if not compatible. """ + type_comatibility = self.compare(second_object) + if ( + type_comatibility != NeuralTypeComparisonResult.SAME + and type_comatibility != NeuralTypeComparisonResult.GREATER + ): + raise NeuralPortNmTensorMismatchError(parent_type_name, port_name, self, second_object, type_comatibility) + @staticmethod def __check_sanity(axes): # check that list come before any tensor dimension @@ -188,9 +196,6 @@ def __compare_axes(axes_a, axes_b) -> int: return 3 -ModulePort = namedtuple('ModulePort', ["name", "port"]) - - class NmTensor(NeuralType): """Class representing data which flows between NeuralModules' ports. It also has a type of NeuralType represented by inheriting from NeuralType @@ -244,16 +249,16 @@ def consumers_ports(self): """ return self._consumers_ports - def add_consumer(self, module, input_port_name): + def add_consumer(self, module_name, input_port_name): """ Adds tensor "consumer". Args: - module: Module that accepts the tensor as input. + module_name: Name of the module that accepts the tensor as input. input_port_name: Name of the module's input port. """ - self._consumers_ports.append(ModulePort(module.name, input_port_name)) + self._consumers_ports.append(ModulePort(module_name, input_port_name)) @property def type(self): @@ -319,20 +324,15 @@ class NeuralPortNameMismatchError(NeuralTypeError): """Exception raised when neural module is called with incorrect port names.""" - def __init__(self, message): - self.message = message + def __init__(self, input_port_name): + self.message = "Wrong input port name: {0}".format(input_port_name) class NeuralPortNmTensorMismatchError(NeuralTypeError): """Exception raised when a port is fed with a NmTensor of incompatible type.""" - def __init__(self, message): - self.message = message - - -class CanNotInferResultNeuralType(NeuralTypeError): - """Exception raised when NeuralType of output can not be inferred.""" - - def __init__(self, message): - self.message = message + def __init__(self, class_name, port_name, first_type, second_type, type_comatibility): + self.message = "\nIn {}. \nPort: {} and a NmTensor it was fed are \n".format(class_name, port_name) + self.message += "of incompatible neural types:\n\n{} \n\n and \n\n{}".format(first_type, second_type) + self.message += "\n\nType comparison result: {}".format(type_comatibility) diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 96e808d91ddb..437cbc1df88b 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -25,4 +25,6 @@ from .helpers import * from nemo.utils.app_state import AppState from nemo.utils.object_registry import ObjectRegistry +from nemo.utils.module_port import ModulePort +from nemo.utils.bound_inputs import BoundInputs, BoundInput from nemo.utils.bound_outputs import BoundOutputs diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index 74ad259d9215..23ae2783cf18 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -22,7 +22,9 @@ class Singleton(type): - """ Implementation of a generic singleton meta-class. """ + """ Implementation of a generic, tread-safe singleton meta-class. + Can be used as meta-class, i.e. will create + """ # List of instances - one per class. __instances = {} @@ -61,8 +63,9 @@ def __init__(self, device=None): self._device = nemo.core.DeviceType.GPU else: self._device = device - # Create registers. + # Create module registry. self._module_registry = nemo.utils.ObjectRegistry("module") + # Create graph manager (registry with some additional functionality). self._neural_graph_manager = nemo.core.NeuralGraphManager() @property diff --git a/nemo/utils/bound_inputs.py b/nemo/utils/bound_inputs.py new file mode 100644 index 000000000000..10056f3a5de5 --- /dev/null +++ b/nemo/utils/bound_inputs.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from collections import namedtuple +from collections.abc import MutableMapping + +from nemo.utils import logging +from nemo.utils.module_port import ModulePort + +# Actually this throws error as module dependency is: core depends on utils :] +# from nemo.core.neural_types import NeuralType + + +class BoundInput(object): + """ A helper class represenging a single bound input. """ + + def __init__(self, type, modules=[]): + """ + Initializes object. + + Args: + type: a NeuralType object. + modules: a list of ModulePort tuples (module name, port name). + """ + self._type = type + self._modules = modules + + def bind(self, modules): + """ Binds the modules to this "graph input". + + Args: + modules: List of ModulePort tuples to be added. + """ + for module in modules: + self._modules.append(module) + + @property + def type(self): + """ Returns NeuralType of that input. """ + return self._type + + @property + def modules(self): + """ Returns list of bound modules (i.e. (module name, port name) tupes) """ + return self._modules + + +class BoundInputs(MutableMapping): + ''' + A specialized dictionary that contains bound inputs of a Neural Graph. + ''' + + def __init__(self): + """ Initializes the mapping. """ + self._inputs = {} + + def __setitem__(self, key, value): + """ + This method is used to "create" a bound input, i.e. copy definition from indicated module input port. + + Args: + key: name of the input port of the Neural Graph. + value: NeuralType that will be set. + """ + if key in self._inputs.keys(): + raise KeyError("Overwriting definition of a previously bound port `{}` is not allowed".format(key)) + # Make sure that a proper NeuralType definition was passed here. + # if type(value) is not NeuralType: + # raise TypeError("Port `{}` definition must be must be a NeuralType".format(key)) + + # Ok, add definition to list of mapped (module, port)s. + # Note: for now, there are no mapped modules, thus []. + self._inputs[key] = BoundInput(type=value, modules=[]) + + def __getitem__(self, key): + """ Returns bound input. """ + return self._inputs[key] + + def __delitem__(self, key): + raise NotImplementedError("Deleting a bound input port is not allowed") + + def __iter__(self): + """ Iterates over the bound inputs. """ + return iter(self._inputs) + + def __len__(self): + """ Return number of bound inputs. """ + return len(self._inputs) + + @property + def definitions(self): + """ Property returns definitions of the input ports by extracting them on the fly from list. """ + # Extract port definitions (Neural Types) from the inputs list. + return {k: v.type for k, v in self._inputs.items()} diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index cd18f38af31e..37907dfa18d9 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -23,16 +23,20 @@ class BoundOutputs(MutableMapping): ''' - A specialized dictionary that contains bound outputs. - In fact stores two lists of bound tensors ("default" and "manual"), and accesses them following the logic: - 1) Record all output tensors as "default" - 2) If user doesn't set any outputs manually, use the default. + A specialized dictionary that contains bound outputs of a Neural Graph. + In fact stores three lists of bound tensors: + - "all" output tensors of all modules within a graph, + - "default" output tensors with default keys taken from outputs of modules (might result in + overwriting some keys), and + - "manual" used for specifying the subset of output tensors, each with a new/different key + When accessing the output tensors, it returns the "manual" tensors. If "manual" tensors are not defined, + will return/work on "default" tensors. ''' def __init__(self): - """ Initializes two dictionaries. """ + """ Initializes three (empty) dictionaries. """ # This dictionary stores list of output tensors of all modules, one key per - # This will generate a "all output tensors" dictionary, where the key is name of "producer" module, + # This will generate a "all output tensors" dictionary, where the key is name of "producer" module, # and the value contains all produced tensors. self._all_tensors = {} @@ -76,8 +80,8 @@ def __len__(self): else: # Use default dict. return len(self._default_tensors) - def bind_defaults(self, tensors_list): - """ Binds default output tensors. + def bind(self, tensors_list): + """ Binds the output tensors. Args: tensors_list: List of tensors to be added. @@ -103,7 +107,7 @@ def bind_defaults(self, tensors_list): @property def all(self): - """ Returns dictionary of dictionary of output tensors. """ + """ Returns dictionary of all output tensors. """ return self._all_tensors @property diff --git a/nemo/utils/module_port.py b/nemo/utils/module_port.py new file mode 100644 index 000000000000..c85d9a63b622 --- /dev/null +++ b/nemo/utils/module_port.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +__all__ = [ + 'ModulePort', +] + +from collections import namedtuple + +# Tuple used for storing "module name" and its "port name". +# (used in NmTensor's producer/consumer, port binding etc.). +ModulePort = namedtuple('ModulePort', ["module_name", "port_name"]) diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index bfe405e23214..018dd3dd555c 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -83,33 +83,33 @@ def test_nm_tensors_producer_consumers(self): # Check tensor x producer and consumers. p = x.producer_port cs = x.consumers_ports - assert p.name == "source" - assert p.port == "x" + assert p.module_name == "source" + assert p.port_name == "x" assert len(cs) == 1 - assert cs[0].name == "tm" - assert cs[0].port == "x" + assert cs[0].module_name == "tm" + assert cs[0].port_name == "x" # Check tensor y producer and consumers. p = y.producer_port cs = y.consumers_ports - assert p.name == "source" - assert p.port == "y" + assert p.module_name == "source" + assert p.port_name == "y" assert len(cs) == 2 - assert cs[0].name == "loss" - assert cs[0].port == "target" - assert cs[1].name == "loss2" - assert cs[1].port == "target" + assert cs[0].module_name == "loss" + assert cs[0].port_name == "target" + assert cs[1].module_name == "loss2" + assert cs[1].port_name == "target" # Check tensor y_pred producer and consumers. p = y_pred.producer_port cs = y_pred.consumers_ports - assert p.name == "tm" - assert p.port == "y_pred" + assert p.module_name == "tm" + assert p.port_name == "y_pred" assert len(cs) == 2 - assert cs[0].name == "loss" - assert cs[0].port == "predictions" - assert cs[1].name == "loss2" - assert cs[1].port == "predictions" + assert cs[0].module_name == "loss" + assert cs[0].port_name == "predictions" + assert cs[1].module_name == "loss2" + assert cs[1].port_name == "predictions" @pytest.mark.unit def test_nm_tensors_types(self): From 9c3f78bea0addfa69e8539efd57daaade0db301f Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 15 Apr 2020 11:36:18 -0700 Subject: [PATCH 040/106] bound output test fix Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests1.py | 1 - .../graph_composition_integration_tests2_1.py | 18 ++++++++++-------- nemo/core/neural_graph.py | 7 +++++++ tests/unit/core/test_neural_graph_nesting.py | 1 + tests/unit/utils/test_bound_outputs.py | 6 +++--- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index 1bf11a732ab8..14005b00a2fc 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -35,7 +35,6 @@ lss = loss(predictions=p, target=t) # Manual bind. g0.output_ports["output"] = loss - # print(g0.output_ports) # print(g0.output_ports["x"]) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index 172583114656..ca68a5a5070a 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -23,10 +23,10 @@ nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32) -m1 = TaylorNet(dim=4) -m2 = TaylorNet(dim=4) -loss = MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") +m1 = TaylorNet(dim=4, name="m1") +m2 = TaylorNet(dim=4, name="m2") +loss = MSELoss(name="loss") logging.info( "This example shows how one can nest one graph into another - with manual binding of selected output ports." @@ -38,14 +38,16 @@ prediction1 = m1(x=x) prediction2 = m2(x=x) # Manually bind the selected output ports. - g1.output_ports["x"] = x - g1.output_ports["t"] = t - g1.output_ports["prediction"] = prediction1 + g1.output_ports["ix"] = x + g1.output_ports["te"] = t + g1.output_ports["prediction"] = prediction2 -with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g11: +with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g2: x1, t1, p1 = g1() lss = loss(predictions=p1, target=t1) +import pdb; pdb.set_trace() + # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index fb05285e2a41..2c6b7ee0299e 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -183,9 +183,16 @@ def __call__(self, **kwargs): ###### PRODUCE OUTPUTS. ###### # This part is different from Neural Module. # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! + # Still, those must be bound in the outer (active) graph. if len(self._bound_outputs) == 1: # Return the single tensor. results = next(iter(self._bound_outputs.values())) + + # "Copy" all the tensors from the nested graph. TODO COPY?? + # Bind the "default" output ports. + self._app_state.active_graph.bind_outputs([results]) + + else: # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 9d6abadfa28f..2bb6736a9d57 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -108,6 +108,7 @@ def test_graph_nesting_possible_operation_modes(self): with NeuralGraph(operation_mode=OperationMode.both): _, _ = inference() + @pytest.mark.unit def test_output_ports_binding(self): # Create modules. data_source = RealFunctionDataLayer(n=100, batch_size=1) diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index 28063859af3f..5e1fac351368 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -40,9 +40,9 @@ def test_binding(self): # Test default binding. bound_outputs = BoundOutputs() - bound_outputs.bind_defaults([x, y]) - bound_outputs.bind_defaults([y_pred]) - bound_outputs.bind_defaults([lss]) + bound_outputs.bind([x, y]) + bound_outputs.bind([y_pred]) + bound_outputs.bind([lss]) # Delete not allowed. with pytest.raises(NotImplementedError): From b8bf040ca75861f00b6ed3dc9d41f741d4dd90e1 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 15 Apr 2020 12:22:45 -0700 Subject: [PATCH 041/106] preparing ground for tensor copy Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph.py | 45 ++++++++++++++++---- nemo/core/neural_modules.py | 2 +- nemo/core/neural_types/neural_type.py | 25 ++++++++--- tests/unit/core/test_neural_graph_nesting.py | 19 +++++++++ 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 2c6b7ee0299e..9ffea3b754c1 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -16,7 +16,7 @@ # limitations under the License. # ============================================================================= -from collections import namedtuple +from collections import namedtuple, OrderedDict from typing import Dict, Optional from nemo.core import OperationMode @@ -68,7 +68,7 @@ def __init__(self, operation_mode=OperationMode.both, name=None): # "Modules" - list of modules constituting "nodes" in a given graph. self._modules = {} # "Steps": order of the execution of modules in a graph. - self._steps = [] + self._steps = OrderedDict() def __call__(self, **kwargs): """ @@ -97,10 +97,8 @@ def __call__(self, **kwargs): # Get input and output ports definitions. input_port_defs = self.input_ports - # "Copy" all the operations from the previous graph. TODO better! - for step in self._steps: - self._app_state.active_graph.record_step(*step) - # print(self._steps) + # "Nest" this graph into active graph. + self._app_state.active_graph.copy(self) ###### PROCESS INPUTS. ###### # Iterate through all passed parameters. @@ -235,6 +233,16 @@ def output_ports(self): """ return self._bound_outputs + @property + def modules(self): + """ Returns modules. """ + return self._modules + + @property + def steps(self): + """ Returns steps. """ + return self._steps + @property def operation_mode(self): """ Returns operation mode. """ @@ -295,7 +303,7 @@ def list_modules(self): desc += " * `{}` ({})\n".format(key, value) return desc - def record_step(self, module, inputs): + def record_step(self, module): """ Records the operation (module plus passed inputs) on a list. """ @@ -305,8 +313,8 @@ def record_step(self, module, inputs): # Add module to list of modules. self._modules[module.name] = module - # Add step. - self._steps.append([module, inputs]) + # Add step - store the module name. + self._steps[len(self._steps)] = module.name # def bind_input(self, port_name, port_definition, bound_module): # # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) @@ -318,6 +326,25 @@ def record_step(self, module, inputs): # # Additionally, remember the bound module # self._bound_input_modules[port_name] = bound_module + def copy(self, graph): + """ + Method copies a graph: modules, steps, topology (tensors). + + Args: + graph: Graph to be copied (is "nested" in this graph). + """ + # "Copy" modules. + for key,module in graph.modules.items(): + self._modules[key] = module + + # "Copy" steps, i.e. append them to the list, following the original order. + for key, step in graph.steps.items(): + # Add step - store the module name. + self._steps[len(self._steps)] = step + + # Copy tensors - those will produce "real" copies of the objects. + + def bind_outputs(self, tensors_list): """ Binds the output tensors. diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index ed51dc13ff18..b02e9d0e66dc 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -455,7 +455,7 @@ def __call__(self, **kwargs): output_port_defs = self.output_ports # Record the operation (i.e. add a single module). - self._app_state.active_graph.record_step(self, kwargs.items()) + self._app_state.active_graph.record_step(self) ###### PROCESS INPUTS. ###### # Iterate through all passed parameters. diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 445c17d02710..29f1b08a42ba 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -28,6 +28,7 @@ from nemo.core.neural_types.axes import AxisKind, AxisType from nemo.core.neural_types.comparison import NeuralTypeComparisonResult from nemo.core.neural_types.elements import * +from nemo.utils.app_state import AppState from nemo.utils.module_port import ModulePort @@ -210,7 +211,11 @@ def __init__(self, producer, producer_args, name, ntype=None): of arguments which were sent to producer to create this """ super(NmTensor, self).__init__(axes=ntype.axes, elements_type=ntype.elements_type, optional=ntype.optional) - self._producer = producer + # producer is None: a special case present in some of the unit tests. + if producer is None: + self._producer_name = "None" + else: + self._producer_name = producer.name self._producer_args = producer_args self._output_port_name = name self._uuid = str(uuid.uuid4()) @@ -223,7 +228,7 @@ def producer(self): Returns: NeuralModule object which produced this NmTensor. """ - return self._producer + return AppState().modules[self._producer_name] @property def producer_name(self): @@ -231,7 +236,7 @@ def producer_name(self): Returns: Name of the producer of the tensor. """ - return self._producer.name + return self._producer_name @property def producer_port(self): @@ -239,7 +244,7 @@ def producer_port(self): Returns: A tuple containing producer name and corresponding output port name. """ - return ModulePort(self._producer.name, self._output_port_name) + return ModulePort(self._producer_name, self._output_port_name) @property def consumers_ports(self): @@ -268,6 +273,14 @@ def type(self): """ return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) + + def copy(self): + """ + Returns: + A copy of the current the current + """ + return 1 + # def serialize(self): # """ # Serializes tensor to a dictionary (yaml structure). @@ -309,9 +322,9 @@ def unique_name(self): Returns: str: unique name """ - if self._producer is None: + if self._producer_name is None: raise ValueError("This NmTensor does not have a unique name") - return f"{self._output_port_name}~~~{self.producer.name}~~~{self._uuid}" + return f"{self._output_port_name}~~~{self._producer_name}~~~{self._uuid}" class NeuralTypeError(Exception): diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 2bb6736a9d57..5ed1f9d50e86 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -136,3 +136,22 @@ def test_output_ports_binding(self): assert len(g1.output_ports) == 2 assert g1.output_ports["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert g1.output_ports["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_graph_nesting_topology_copy_one_module_defaults(self): + """ Test whether when nesting one graph into another the graph topology (tensors) will be copied. """ + + dl = RealFunctionDataLayer(n=100, batch_size=32, name="t1_dl") + + with NeuralGraph(operation_mode=OperationMode.training, name="t1_g1") as g1: + xg1, tg1 = dl() + + with NeuralGraph(operation_mode=OperationMode.training, name="t1_g2") as g2: + xg2, tg2 = g1() + + # We expect that both graphs will have the same modes/steps. + assert len(g1.steps) == len(g2.steps) + assert g1.steps[0] == g2.steps[0] + assert len(g1) == len(g2) + assert g1["t1_dl"] is g2["t1_dl"] + \ No newline at end of file From 691e90b2ef9261bb98d87a410c2c051c9b5c242c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 17 Apr 2020 20:26:36 -0700 Subject: [PATCH 042/106] Rewritten graph nesting - works by executing inner graph modules' call() - operational, all examples working Signed-off-by: Tomasz Kornuta --- ...osition_integration_tests0_jasper_named.py | 93 +++++ .../graph_composition_integration_tests1.py | 2 +- .../graph_composition_integration_tests2.py | 6 +- .../graph_composition_integration_tests2_0.py | 52 +++ .../graph_composition_integration_tests2_1.py | 5 +- .../graph_composition_integration_tests3.py | 10 +- .../graph_composition_integration_tests3_1.py | 4 +- nemo/core/neural_graph.py | 394 +++++++++++------- nemo/core/neural_modules.py | 7 +- nemo/core/neural_types/neural_type.py | 26 +- nemo/utils/bound_inputs.py | 31 +- nemo/utils/bound_outputs.py | 131 +++--- nemo/utils/module_port.py | 5 + tests/unit/core/test_neural_graph_nesting.py | 1 - 14 files changed, 537 insertions(+), 230 deletions(-) create mode 100644 examples/start_here/graph_composition_integration_tests0_jasper_named.py create mode 100644 examples/start_here/graph_composition_integration_tests2_0.py diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py new file mode 100644 index 000000000000..c2962b06d2ff --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -0,0 +1,93 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from functools import partial +from os.path import expanduser + +from ruamel.yaml import YAML + +import nemo +import nemo.collections.asr as nemo_asr +from nemo.collections.asr.helpers import monitor_asr_train_progress +from nemo.core import NeuralGraph +from nemo.utils.app_state import AppState +from nemo.utils import logging + +nf = nemo.core.NeuralModuleFactory() +app_state = AppState() + +logging.info( + "This example shows how one can build a Jasper model using the `default` (implicit) graph." + F" This approach works for applications containing a single graph." +) + +# Set paths to "manifests" and model configuration files. +train_manifest = "~/TestData/an4_dataset/an4_train.json" +val_manifest = "~/TestData/an4_dataset/an4_val.json" +model_config_file = "~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" + +yaml = YAML(typ="safe") +with open(expanduser(model_config_file)) as f: + jasper_params = yaml.load(f) +# Get vocabulary. +vocab = jasper_params['labels'] + +# Create neural modules. +data_layer = nemo_asr.AudioToTextDataLayer.import_from_config( + model_config_file, + "AudioToTextDataLayer_train", + overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, +) + +data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.import_from_config( + model_config_file, "AudioToMelSpectrogramPreprocessor" +) + +jasper_encoder = nemo_asr.JasperEncoder.import_from_config(model_config_file, "JasperEncoder") +jasper_decoder = nemo_asr.JasperDecoderForCTC.import_from_config( + model_config_file, "JasperDecoderForCTC", overwrite_params={"num_classes": len(vocab)} +) +ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) +greedy_decoder = nemo_asr.GreedyCTCDecoder() + +# Create the Jasper composite module. +with NeuralGraph() as Jasper: + i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. + i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) + i_log_probs = jasper_decoder(encoder_output=i_encoded) # All output ports are bind (for now!) + +with NeuralGraph(name="training") as training: + # Create the "implicit" training graph. + o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() + # Use Jasper module as any other neural module. + o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = Jasper(input_signal=o_audio_signal, length=o_audio_signal_len) + o_predictions = greedy_decoder(log_probs=o_log_probs) + o_loss = ctc_loss(log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len) + tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] + +train_callback = nemo.core.SimpleLossLoggerCallback( + tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) +) +#import pdb;pdb.set_trace() +nf.train( + tensors_to_optimize=[o_loss], + optimizer="novograd", + callbacks=[train_callback], + optimization_params={"num_epochs": 50, "lr": 0.01}, +) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index 14005b00a2fc..e8e57ad114ac 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -34,7 +34,7 @@ p = m2(x=x) lss = loss(predictions=p, target=t) # Manual bind. - g0.output_ports["output"] = loss + g0.output_ports["output"] = lss # print(g0.output_ports) # print(g0.output_ports["x"]) diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py index ad0b234faf9c..3724ae002c65 100644 --- a/examples/start_here/graph_composition_integration_tests2.py +++ b/examples/start_here/graph_composition_integration_tests2.py @@ -23,9 +23,9 @@ nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32) -m2 = TaylorNet(dim=4) -loss = MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") +m2 = TaylorNet(dim=4, name="m2") +loss = MSELoss(name="loss") logging.info( "This example shows how one can nest one graph into another - with binding of output ports." diff --git a/examples/start_here/graph_composition_integration_tests2_0.py b/examples/start_here/graph_composition_integration_tests2_0.py new file mode 100644 index 000000000000..317012727741 --- /dev/null +++ b/examples/start_here/graph_composition_integration_tests2_0.py @@ -0,0 +1,52 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback +from nemo.utils import logging + +nf = NeuralModuleFactory(placement=DeviceType.CPU) +# Instantiate the necessary neural modules. +dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") +m1 = TaylorNet(dim=4, name="m1") +loss = MSELoss(name="loss") + +logging.info( + "This example shows how one can nest one graph into another - with manual binding of selected output ports." + F" Please note that the nested graph can be used exatly like any other module." +) + +with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: + xg1, tg1 = dl() + +with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: + xg2, tg2 = g1() + pg2 = m1(x=xg2) + lssg2 = loss(predictions=pg2, target=tg2) + + +#import pdb;pdb.set_trace() + +# SimpleLossLoggerCallback will print loss values to console. +callback = SimpleLossLoggerCallback( + tensors=[lssg2], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +) + +# Invoke "train" action. +nf.train([lssg2], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index ca68a5a5070a..cb238910ca5f 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -17,6 +17,8 @@ # limitations under the License. # ============================================================================= +import pdb + from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging @@ -46,7 +48,8 @@ x1, t1, p1 = g1() lss = loss(predictions=p1, target=t1) -import pdb; pdb.set_trace() + +pdb.set_trace() # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( diff --git a/examples/start_here/graph_composition_integration_tests3.py b/examples/start_here/graph_composition_integration_tests3.py index 007a99a06fe9..11b3372ae8a3 100644 --- a/examples/start_here/graph_composition_integration_tests3.py +++ b/examples/start_here/graph_composition_integration_tests3.py @@ -23,9 +23,9 @@ nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32) -fx = TaylorNet(dim=4) -loss = MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") +fx = TaylorNet(dim=4, name="fx") +loss = MSELoss(name="loss") logging.info( "This example shows how one can nest one graph into another - with binding of the input ports." @@ -43,8 +43,8 @@ # Add modules to graph. x, t = dl() # Incorporate modules from existing graph. - p = g2(x=x) - lss = loss(predictions=p, target=t) + pred = g2(x=x) + lss = loss(predictions=pred, target=t) # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index d8aa574652d7..944838100e50 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -46,7 +46,9 @@ # Add modules to graph. x, t = dl() # Incorporate modules from the existing graph. - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() p = g2(input=x) lss = loss(predictions=p, target=t) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 9ffea3b754c1..deb5579dcc16 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -16,15 +16,12 @@ # limitations under the License. # ============================================================================= -from collections import namedtuple, OrderedDict +from collections import OrderedDict, namedtuple from typing import Dict, Optional from nemo.core import OperationMode from nemo.core.neural_interface import NeuralInterface -from nemo.core.neural_types import ( - NeuralPortNameMismatchError, - NmTensor, -) +from nemo.core.neural_types import NeuralPortNameMismatchError, NmTensor from nemo.utils.bound_inputs import BoundInput, BoundInputs from nemo.utils.bound_outputs import BoundOutputs @@ -52,23 +49,25 @@ def __init__(self, operation_mode=OperationMode.both, name=None): # Store name and operation mode. self._operation_mode = operation_mode - # Input ports and tensors - empty for now. - self._bound_input_ports = {} - self._bound_input_tensors = {} - # List of modules of bound inputs - so we will update their output tensors when the "bound" - # input port will be connected. - self._bound_input_modules = {} + # "Modules" - list of modules constituting "nodes" in a given graph. + self._modules = {} + + # All tensors produced within this graph (dict of dicts). + # This stores "all output tensors" dictionary, where the key is the name of "producer" module, + # and the value contains a dictionary of all tensors produced by it. + self._all_tensors = {} + + # "Steps": order of the execution of modules in a graph. + self._steps = OrderedDict() # Bound inputs. self._bound_inputs = BoundInputs() # Bound outputs. - self._bound_outputs = BoundOutputs() + self._bound_outputs = BoundOutputs(self._all_tensors) - # "Modules" - list of modules constituting "nodes" in a given graph. - self._modules = {} - # "Steps": order of the execution of modules in a graph. - self._steps = OrderedDict() + # Flag indicating whether the "default" output ports/tensors will be automatically bound. + self.default_output_binding = True def __call__(self, **kwargs): """ @@ -94,112 +93,16 @@ def __call__(self, **kwargs): if inner_mode == OperationMode.inference and outer_mode == OperationMode.both: raise TypeError("Cannot nest 'inference' graph into 'both'") - # Get input and output ports definitions. - input_port_defs = self.input_ports - - # "Nest" this graph into active graph. - self._app_state.active_graph.copy(self) - - ###### PROCESS INPUTS. ###### - # Iterate through all passed parameters. + # Check inputs: iterate through all inputs passed to the "self". for port_name, port_content in kwargs.items(): - # make sure that passed arguments correspond to input port names - if port_name not in input_port_defs.keys(): + # Make sure that passed arguments correspond to input port names. + if port_name not in self.input_ports.keys(): raise NeuralPortNameMismatchError(port_name) - # Analogically to NeuralModule, at that point the input can be one of three types: - # * NeuralGraph -> bind port using the default name and type. - # * BoundInput -> check definition, if ok bind port - # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). - - # Check what was actually passed. - if type(port_content) is NeuralGraph: - - # Make sure that port_content is the currently active graph! - if port_content is not self._app_state.active_graph: - raise ConnectionError("Ports can be bound only by passing the active graph object!") - - # This case: we are nesting one graph into another and must bind input port of one graph in another! - # So generally we will "copy" the BoundInput object. - - # Create an alias so the logic will be more clear. - active_graph = port_content - - # Copy the port "definition" (i.e. is NeuralType) using the same port name. - # This might throw an exception if port with that name was already bound! - active_graph.input_ports[port_name] = input_port_defs[port_name].type - - # Bind the neural graph input port, i.e. remember that a given graph port should pass data - # to all "bound modules" (when it finally will be connected). - active_graph.input_ports[port_name].bind(input_port_defs[port_name].modules) - - # Please note that there are no "consumers" here - this is a "pure" binding. - - elif type(port_content) is BoundInput: - - # Compare input port definition with the received definition. - input_port_defs[port_name].type.compare_and_raise_error( - self.__class__.__name__, port_name, port_content.type - ) - - # Bind the input port modules, i.e. remember that a given graph port should pass data - # to all "bound modules" (when it finally will be connected). - port_content.bind(self.modules) - - # Please note that there are no "consumers" here - this is a "pure" binding. - - elif type(port_content) is NmTensor: - # Compare input port definition with the received definition. - input_port_defs[port_name].type.compare_and_raise_error( - self.__class__.__name__, port_name, port_content - ) - - # Reaching that point means that we accepted input to a bound graph port. - # Need to connect it - "copy" all modules connected to this "bound input" as consumers. - for consumer in input_port_defs[port_name].modules: - # Add consumer. - port_content.add_consumer(consumer.module_name, consumer.port_name) - - # The current graph parsing requires us to update all outputs of - # a module that "accepted" the input. - # In other words: for every "consumer" we need to update all tensors it produced. - # Update means changing the original producer_args for ALL TENSORS IN THE GRAPH produced by - # this module. - producer_name = consumer.module_name - if producer_name in self._bound_outputs.all.keys(): - # Get all tensor producer by this module. - for output_tensor in self._bound_outputs.all[producer_name]: - # Set "input port value" to new content - which indicates tensor (and producer) - # that will be used during graph backward traverse. - output_tensor.producer_args[port_name] = port_content # i.e. Tensor. - - else: - raise TypeError( - "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) - ) - - ###### PRODUCE OUTPUTS. ###### - # This part is different from Neural Module. - # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! - # Still, those must be bound in the outer (active) graph. - if len(self._bound_outputs) == 1: - # Return the single tensor. - results = next(iter(self._bound_outputs.values())) - - # "Copy" all the tensors from the nested graph. TODO COPY?? - # Bind the "default" output ports. - self._app_state.active_graph.bind_outputs([results]) - - - else: - # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). - output_class_name = f'{self.__class__.__name__}Output' - result_type = namedtuple(typename=output_class_name, field_names=self._bound_outputs.keys()) - - # Return the "default" bound output ports. - results = result_type(*self._bound_outputs.values()) + # "Nest" this graph into an active graph. + results = self._app_state.active_graph.nest(self, kwargs) - # Return the results. + # Return output tensors. return results @property @@ -308,6 +211,8 @@ def record_step(self, module): Records the operation (module plus passed inputs) on a list. """ # Check if module with that name already exists. + # TODO: Uncomment when we will refactor all examples so training/validation graphs won't be added + # to the "default" graph. # if module.name in self._modules.keys(): # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) # Add module to list of modules. @@ -316,34 +221,215 @@ def record_step(self, module): # Add step - store the module name. self._steps[len(self._steps)] = module.name - # def bind_input(self, port_name, port_definition, bound_module): - # # print("Binding input: `{}`: def = `{}` value = NONE".format(port_name, port_definition)) - # # Copy the definition of the port to graph input port definition. - # self._bound_input_ports[port_name] = port_definition + def unused_input_processing(self): + """ + Old implementation! To be deleted!!!! Wrrrrrr!!! + """ + # Check inputs: iterate through all inputs passed to the inner graph. + for port_name, port_content in inner_graph_args.items(): + # Make sure that passed arguments correspond to input port names. + if port_name not in inner_graph.input_ports.keys(): + raise NeuralPortNameMismatchError(port_name) + + # Analogically to NeuralModule, at that point the input can be one of three types: + # * NeuralGraph -> bind port using the default name and type. + # * BoundInput -> check definition, if ok bind port. + # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). + + # Check what was actually passed. + if type(port_content) is NeuralGraph: + + # Make sure that port_content is the currently active graph, i.e. THIS GRAPH! + if port_content is not self: + raise ConnectionError("Ports can be bound only by passing the active graph object!") + + # This case: we are nesting one graph into another and must bind input port of one graph in another! + # So generally we will "copy" the BoundInput object, using the same name. + + # Copy the port "definition" (i.e. is NeuralType) using the same port name. + # This might throw an exception if port with that name was already bound! + self.input_ports[port_name] = inner_graph.input_ports[port_name].type + + # Remember that a given graph port should pass data to all "bound modules" of the "nested" graph + # (when it finally will be connected). + self.input_ports[port_name].bind(inner_graph.input_ports[port_name].modules) - # # Indicate that this tensor is missing and has to be provided! - # self._bound_input_tensors[port_name] = None - # # Additionally, remember the bound module - # self._bound_input_modules[port_name] = bound_module + # Please note that there are no "consumers" here - this is a "pure" binding. - def copy(self, graph): + elif type(port_content) is BoundInput: + # Check if BoundInput belongs to the outer graph (i.e. self)! + own_port = False + for port in self.input_ports.items(): + if port is BoundInput: + own_port = True + break + if not own_port: + raise NeuralPortNameMismatchError(port_name) + + # Compare input port definition with the received definition. + port_content.type.compare_and_raise_error( + self.__class__.__name__, port_name, inner_graph.input_ports[port_name].type + ) + + # Remember that a given graph port should pass data to all "bound modules" of the "nested" graph + # (when it finally will be connected). + port_content[port_name].bind(inner_graph.input_ports[port_name].modules) + + # Please note that there are no "consumers" here - this is a "pure" binding. + + elif type(port_content) is NmTensor: + + # Compare input port definition onf the inner graph with the received definition. + inner_graph.input_ports[port_name].type.compare_and_raise_error( + self.__class__.__name__, port_name, port_content.type + ) + # Note that this tensor is already! a part of this graph. + # (It has to be output of one of the previous modules.) + # So we can find it in: self._tensors + + # Reaching that point means that we accepted input to a bound graph port. + # Need to connect it - "copy" all modules connected to this "bound input" as consumers. + for consumer in inner_graph.input_ports[port_name].modules: + # Add consumer. + port_content.add_consumer(consumer.module_name, consumer.port_name) + + # The current graph parsing requires us to update all outputs of + # a module that "accepted" the input. + # In other words: for every "consumer" we need to update all tensors it produced. + # Update means changing the original producer_args for ALL TENSORS IN THE GRAPH produced by + # this module. + # producer_name = consumer.module_name + # if producer_name in self._tensors.keys(): + # # Get all tensor producer by this module. + # for output_tensor in self._tensors[producer_name]: + # # Set "input port value" to new content - which indicates tensor (and producer) + # # that will be used during graph backward traverse. + # output_tensor.producer_args[port_name] = port_content # i.e. Tensor. + + else: + raise TypeError( + "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) + ) + + def nest(self, inner_graph, inner_graph_args): """ - Method copies a graph: modules, steps, topology (tensors). + Method nests (copies) a graph: modules, steps, topology (tensors). Args: - graph: Graph to be copied (is "nested" in this graph). + inner_graph: Graph to be copied (will be "nested" in this (self) graph). + inner_graph_args: inputs passed to the graph call. """ # "Copy" modules. - for key,module in graph.modules.items(): - self._modules[key] = module - - # "Copy" steps, i.e. append them to the list, following the original order. - for key, step in graph.steps.items(): + for key, module in inner_graph.modules.items(): + # Check if module with that name already exists. + # TODO: Uncomment when we will refactor all examples so training/validation graphs won't be added + # to the "default" graph. + # if key in self._modules.keys(): + # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) + self._modules[key] = module + + # Next we should copy the topography - i.e. produce "real" copies of tensors. + # In fact, instead of copying, we will produce them, following: + # - the execution order defined in "steps" + # - connectivity defined in tensor' consumers-ports + # (so the same logic that will be used in graph deserialization) + + # So let us first serialize the connections of the nested graph. + # Create a list: (producer.port -> consumer.port) + inner_connections = [] + for tensors in inner_graph.tensors.values(): + for t in tensors.values(): + inner_connections.extend(t.connections()) + + # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce + # them only for ports that are already indicated as the "bound" ones in the inner graph. + self.default_output_binding = False + + # Now "copy" steps, i.e. append them to the list, following the original order. + # Moreover, actually execute each step. + for _, module_name in inner_graph.steps.items(): # Add step - store the module name. - self._steps[len(self._steps)] = step + self._steps[len(self._steps)] = module_name + + # Get the module. + module = self._modules[module_name] + + # Produce list of arguments that will be passed to a given modules. + module_args = {} + # Do it by: + # - harvesing input port names of a given module, + # - checking if the input was not bound (in the inner graph), + # - checking if we have already tensors leading to that input (in outer graph). + for input_port_name in module.input_ports.keys(): + # Check if this port was bound in the inner graph. + key = inner_graph.input_ports.has_binding(module_name, input_port_name) + # If so, then we must pass whatever was passed to that port in the list of arguments. + if key is not None: + module_args[input_port_name] = inner_graph_args[key] + # As a result, the "module" call() will bind this input! + continue + + # Else: find a tensor that should be passed to the given module's input. + # Search for producer/port that we should use. + for connection in inner_connections: + if ( + connection.consumer.module_name == module_name + and connection.consumer.port_name == input_port_name + ): + # Got the connection! + producer_name = connection.producer.module_name + producer_port_name = connection.producer.port_name + break + # Now, the tensor is already produced in outer (i.e. this) graph! + module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + + #import pdb;pdb.set_trace() + # Ok, now we have all keyword arguments. We can call() the module. + # This will collect all the produced output tensors and add them to this graph. + module(**module_args) + + # At that point we have all modules, steps and tensors added to outer (self) graph. + # Now we have to prepare the outputs. - # Copy tensors - those will produce "real" copies of the objects. + # This part is different from Neural Module. + # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! + # Still, those must be bound in the outer (active) graph. + # Get list of "the adequate output tensors". + output_tensors = {} + # Iterate through outputs of the inner graph. + for key, tensor in inner_graph._bound_outputs.tensors.items(): + # Find the tensors within this (outer) graph that are outpus by the same producer-port. + producer_name = tensor.producer_name + producer_port_name = tensor.name + # Get adequate tensor from "outer graph" (self). + output_tensors[key] = self.tensors[producer_name][producer_port_name] + + if len(output_tensors) == 1: + # Return a single tensor. + results = next(iter(output_tensors.values())) + + # Bind the "default" output ports of the inner graph as "default" output ports of this graph. + # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. + self._bound_outputs.bind([results]) + + else: + # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). + output_class_name = f'{self.__class__.__name__}Output' + result_type = namedtuple(typename=output_class_name, field_names=output_tensors.keys()) + + # Return the bound output tensors. + results = result_type(*output_tensors.values()) + + # Bind the "default" output ports of the inner graph as "default" output ports of this graph. + # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. + self._bound_outputs.bind(output_tensors.values()) + + # Ok, now we can turn automatic binding on. + self.default_output_binding = True + + # Return the results. + return results def bind_outputs(self, tensors_list): """ Binds the output tensors. @@ -351,22 +437,40 @@ def bind_outputs(self, tensors_list): Args: tensors_list: List of tensors to be bound. """ - self._bound_outputs.bind(tensors_list) + # Add tensors list to of tensors. + for tensor in tensors_list: + # Add tensor to "all" tensors dictionary. + producer_name = tensor.producer_name + if producer_name not in self._all_tensors.keys(): + self._all_tensors[producer_name] = {} + + port_name = tensor.name + # Add tensor. + self._all_tensors[producer_name][port_name] = tensor + + # Bind the tensors as graph outputs. + if self.default_output_binding: + self._bound_outputs.bind(tensors_list) + + @property + def tensors(self): + """ Returns the dictionary of all output tensors, aggregated by modules (key). """ + return self._all_tensors def show_bound_inputs(self): print("bound input ports: ") - for key, value in self._bound_input_ports.items(): - print(" * `{}`: `{}` ({})".format(key, value, type(value))) + # for key, value in self._bound_input_ports.items(): + # print(" * `{}`: `{}` ({})".format(key, value, type(value))) print("bound input tensors: ") - for key, value in self._bound_input_tensors.items(): - print(" * `{}`: `{}` ({})".format(key, value, type(value))) + # for key, value in self._bound_input_tensors.items(): + # print(" * `{}`: `{}` ({})".format(key, value, type(value))) def show_bound_outputs(self): print("bound (default) output ports: ") - for key, value in self._bound_output_ports_default.items(): - print(" * `{}`: `{}` ({})".format(key, value, type(value))) + # for key, value in self._bound_output_ports_default.items(): + # print(" * `{}`: `{}` ({})".format(key, value, type(value))) print("bound (default) output tensors: ") - for key, value in self._bound_output_tensors_default.items(): - print(" * `{}`: `{}` ({})".format(key, value, type(value))) + # for key, value in self._bound_output_tensors_default.items(): + # print(" * `{}`: `{}` ({})".format(key, value, type(value))) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index b02e9d0e66dc..a50e3e871672 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -352,6 +352,9 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) init_params = loaded_config["init_params"] # Update parameters with additional ones. init_params.update(overwrite_params) + # TODO: Add section name as default module name! + #if section_name is not None: + # init_params.update({"name": section_name}) # Create and return the object. obj = mod_obj(**init_params) @@ -519,7 +522,7 @@ def __call__(self, **kwargs): out_type = output_port_defs[out_name] # Create a single returned tensor. - results = NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,) + results = NmTensor(producer=self, producer_args=kwargs, output_port_name=out_name, ntype=out_type,) # Bind the "default" output ports. self._app_state.active_graph.bind_outputs([results]) @@ -527,7 +530,7 @@ def __call__(self, **kwargs): # Create output tensors. output_tensors = [] for out_name, out_type in output_port_defs.items(): - output_tensors.append(NmTensor(producer=self, producer_args=kwargs, name=out_name, ntype=out_type,)) + output_tensors.append(NmTensor(producer=self, producer_args=kwargs, output_port_name=out_name, ntype=out_type,)) # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 29f1b08a42ba..2848d8f5c4fd 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -29,7 +29,7 @@ from nemo.core.neural_types.comparison import NeuralTypeComparisonResult from nemo.core.neural_types.elements import * from nemo.utils.app_state import AppState -from nemo.utils.module_port import ModulePort +from nemo.utils.module_port import Connection, ModulePort class NeuralType(object): @@ -202,7 +202,7 @@ class NmTensor(NeuralType): It also has a type of NeuralType represented by inheriting from NeuralType object.""" - def __init__(self, producer, producer_args, name, ntype=None): + def __init__(self, producer, producer_args, output_port_name, ntype=None): """NmTensor constructor. Args: @@ -214,10 +214,10 @@ def __init__(self, producer, producer_args, name, ntype=None): # producer is None: a special case present in some of the unit tests. if producer is None: self._producer_name = "None" - else: + else: self._producer_name = producer.name self._producer_args = producer_args - self._output_port_name = name + self._output_port_name = output_port_name self._uuid = str(uuid.uuid4()) # List of tuples (consumer name, input port name) self._consumers_ports = [] @@ -273,20 +273,14 @@ def type(self): """ return NeuralType(axes=self.axes, elements_type=self.elements_type, optional=self.optional) - - def copy(self): + def connections(self): """ - Returns: - A copy of the current the current + "Serializes" the tensor to a list of connections (producer/port, consumer/port). """ - return 1 - - # def serialize(self): - # """ - # Serializes tensor to a dictionary (yaml structure). - # """ - # # serialize type. - # return 1 + connections = [] + for cp in self._consumers_ports: + connections.append(Connection(self.producer_port, cp)) + return connections # @classmethod # def deserialize(cls): diff --git a/nemo/utils/bound_inputs.py b/nemo/utils/bound_inputs.py index 10056f3a5de5..7f4e736a1ef7 100644 --- a/nemo/utils/bound_inputs.py +++ b/nemo/utils/bound_inputs.py @@ -29,16 +29,17 @@ class BoundInput(object): """ A helper class represenging a single bound input. """ - def __init__(self, type, modules=[]): + def __init__(self, type): """ Initializes object. Args: type: a NeuralType object. - modules: a list of ModulePort tuples (module name, port name). """ + # (Neural) Type of input. self._type = type - self._modules = modules + # List of ModulePort tuples to which this input links to (module name, port name). + self._consumers = [] def bind(self, modules): """ Binds the modules to this "graph input". @@ -47,7 +48,7 @@ def bind(self, modules): modules: List of ModulePort tuples to be added. """ for module in modules: - self._modules.append(module) + self._consumers.append(module) @property def type(self): @@ -55,9 +56,9 @@ def type(self): return self._type @property - def modules(self): + def consumers_ports(self): """ Returns list of bound modules (i.e. (module name, port name) tupes) """ - return self._modules + return self._consumers class BoundInputs(MutableMapping): @@ -84,8 +85,8 @@ def __setitem__(self, key, value): # raise TypeError("Port `{}` definition must be must be a NeuralType".format(key)) # Ok, add definition to list of mapped (module, port)s. - # Note: for now, there are no mapped modules, thus []. - self._inputs[key] = BoundInput(type=value, modules=[]) + # Note: for now, there are no mapped modules. + self._inputs[key] = BoundInput(type=value) def __getitem__(self, key): """ Returns bound input. """ @@ -107,3 +108,17 @@ def definitions(self): """ Property returns definitions of the input ports by extracting them on the fly from list. """ # Extract port definitions (Neural Types) from the inputs list. return {k: v.type for k, v in self._inputs.items()} + + def has_binding(self, module_name, port_name): + """ + Checks if there is a binding leading to a given module and its given port. + + Returns: + key in the list of the (bound) input ports that leads to a given module/port or None if the binding was + not found. + """ + for key, binding in self._inputs.items(): + for (module, port) in binding.consumers_ports: + if module == module_name and port == port_name: + return key + return None diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index 37907dfa18d9..86731b763eb2 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -20,12 +20,35 @@ from nemo.utils import logging +class BoundOutput(object): + """ A helper class represenging a single bound output. """ + + def __init__(self, type, producer_port): + """ + Initializes object. + + Args: + type: a NeuralType object. + producer_port: a producer ModulePort tuple (module name, port name). + """ + self._type = type + self._producer_port = producer_port + + @property + def type(self): + """ Returns NeuralType of that output. """ + return self._type + + @property + def producer_port(self): + """ Returns producer port (module name, port name) tuple. """ + return self._producer_port + class BoundOutputs(MutableMapping): ''' A specialized dictionary that contains bound outputs of a Neural Graph. - In fact stores three lists of bound tensors: - - "all" output tensors of all modules within a graph, + In fact stores two lists of bound tensors: - "default" output tensors with default keys taken from outputs of modules (might result in overwriting some keys), and - "manual" used for specifying the subset of output tensors, each with a new/different key @@ -33,88 +56,102 @@ class BoundOutputs(MutableMapping): will return/work on "default" tensors. ''' - def __init__(self): - """ Initializes three (empty) dictionaries. """ - # This dictionary stores list of output tensors of all modules, one key per - # This will generate a "all output tensors" dictionary, where the key is name of "producer" module, - # and the value contains all produced tensors. - self._all_tensors = {} + def __init__(self, tensors_list): + """ Initializes two (empty) dictionaries. """ + + # List + self._tensors_list = tensors_list # This dictionary stores the output tensors collected during the "default" tensor recording. # As they are using the default port names, the second/next tensor published on the same port # will overwrite the old one (Warning). - self._default_tensors = {} + self._default_outputs = {} # This dictionary stores list of output tensors of module "manually" indicated by the user. # In this case tring to overwriting the existing ports with new tensors will be forbidden (Exception). - self._manual_tensors = {} + self._manual_outputs = {} def __setitem__(self, key, value): - """ This method is used to set the manual dictionary. """ - if key in self._manual_tensors.keys(): + """ + This method is used to set the manual output - creates a BoundOutput item and adds it to the list. + + Args: + key: name of the output (port). + value: tensor that will be used to create BoundOutput. + """ + # Make sure that user passed a NmTensor. + assert type(value).__name__ == "NmTensor" + if key in self._manual_outputs.keys(): raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) # Ok, set output. - self._manual_tensors[key] = value + self._manual_outputs[key] = BoundOutput(value.type, value.producer_port) def __getitem__(self, key): - """ Returns output - depending whether there are some manual outputs or not. """ - if len(self._manual_tensors) > 0: - return self._manual_tensors[key] + """ Returns BoundOutput - depending whether there are some manual outputs or not. """ + if len(self._manual_outputs) > 0: + return self._manual_outputs[key] else: # Use default dict. - return self._default_tensors[key] + return self._default_outputs[key] def __delitem__(self, key): - raise NotImplementedError("Deleting a bound output port is not allowed") + raise NotImplementedError("Deleting a bound output is not allowed") def __iter__(self): """ Iterates over the outputs - depending whether there are some manual outputs or not. """ - if len(self._manual_tensors) > 0: - return iter(self._manual_tensors) + if len(self._manual_outputs) > 0: + return iter(self._manual_outputs) else: # Use default dict. - return iter(self._default_tensors) + return iter(self._default_outputs) def __len__(self): """ Return number of outputs - depending whether there are some manual outputs or not. """ - if len(self._manual_tensors) > 0: - return len(self._manual_tensors) + if len(self._manual_outputs) > 0: + return len(self._manual_outputs) else: # Use default dict. - return len(self._default_tensors) + return len(self._default_outputs) def bind(self, tensors_list): - """ Binds the output tensors. + """ Binds the default outputs. Args: tensors_list: List of tensors to be added. """ for tensor in tensors_list: - # Check the presence of the port name in "default" dictionary. - name = tensor.name # Use the default port name. - if name in self._default_tensors.keys(): + # Use the name being combination of producer and port names. + name = tensor.producer_name + "_" + tensor.name + # Check the presence of the port name in "default" dictionary - this is rather impossible, but... + if name in self._default_outputs.keys(): logging.warning( "Overwriting the already bound output port `{}` produced by `{}`".format( - name, self._default_tensors[name].producer_name + name, self._default_outputs[name].producer_name ) ) - # Still, overwrite it. - self._default_tensors[name] = tensor - - # Add tensor to "all" tensors dictionary. - producer_name = tensor.producer_name - if producer_name not in self._all_tensors.keys(): - self._all_tensors[producer_name] = [] - # Add tensor. - self._all_tensors[producer_name].append(tensor) - - @property - def all(self): - """ Returns dictionary of all output tensors. """ - return self._all_tensors + # Still, "overwrite" it. + self._default_outputs[name] = BoundOutput(tensor.type, tensor.producer_port) @property def definitions(self): - """ Property returns definitions of the output ports by extracting them on the fly from the bound tensors. """ - # Get the right tensor dictionary. - d = self._manual_tensors if len(self._manual_tensors) > 0 else self._default_tensors + """ Property returns definitions of the output ports by extracting them on the fly from the bound outputs. """ + # Get the right output dictionary. + d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs - # Extract port definitions (Neural Types) straight from tensors. + # Extract port definitions (Neural Types). return {k: v.type for k, v in d.items()} + + @property + def tensors(self): + """ Property returns output tensors by extracting them on the fly from the bound outputs. """ + # Get the right output dictionary. + d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs + + output_tensors = {} + # Get tensors by acessing the producer-ports. + for k, v in d.items(): + producer_name = v.producer_port.module_name + producer_port_name = v.producer_port.port_name + # Find the right output tensor. + tensor = self._tensors_list[producer_name][producer_port_name] + # Add it to the dictionary. + output_tensors[k] = tensor + + return output_tensors diff --git a/nemo/utils/module_port.py b/nemo/utils/module_port.py index c85d9a63b622..9f5a7323d00a 100644 --- a/nemo/utils/module_port.py +++ b/nemo/utils/module_port.py @@ -25,3 +25,8 @@ # Tuple used for storing "module name" and its "port name". # (used in NmTensor's producer/consumer, port binding etc.). ModulePort = namedtuple('ModulePort', ["module_name", "port_name"]) + + +# Tuple used for connection between a single producer and a single consummer consumer. +# (used in NmTensor's producer/consumer, port binding etc.). +Connection = namedtuple('Connection', ["producer", "consumer"]) diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 5ed1f9d50e86..ee11f53b4186 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -154,4 +154,3 @@ def test_graph_nesting_topology_copy_one_module_defaults(self): assert g1.steps[0] == g2.steps[0] assert len(g1) == len(g2) assert g1["t1_dl"] is g2["t1_dl"] - \ No newline at end of file From ec2bfe641d92cdab3da891e8a1190ba3e204db42 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 17 Apr 2020 20:54:58 -0700 Subject: [PATCH 043/106] fixed tests, formatted the code Signed-off-by: Tomasz Kornuta --- ...osition_integration_tests0_jasper_named.py | 12 ++++++---- .../graph_composition_integration_tests2_0.py | 2 +- nemo/core/neural_graph.py | 10 ++++---- nemo/core/neural_modules.py | 6 +++-- nemo/utils/bound_outputs.py | 14 +++++++---- tests/unit/core/test_module_initialization.py | 2 +- tests/unit/core/test_neural_graph_nesting.py | 23 +++++++++++-------- tests/unit/core/test_neural_graphs.py | 6 ++--- tests/unit/utils/test_bound_outputs.py | 12 ++++++---- 9 files changed, 50 insertions(+), 37 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py index c2962b06d2ff..946454573c1d 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -26,8 +26,8 @@ import nemo.collections.asr as nemo_asr from nemo.collections.asr.helpers import monitor_asr_train_progress from nemo.core import NeuralGraph -from nemo.utils.app_state import AppState from nemo.utils import logging +from nemo.utils.app_state import AppState nf = nemo.core.NeuralModuleFactory() app_state = AppState() @@ -76,15 +76,19 @@ # Create the "implicit" training graph. o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() # Use Jasper module as any other neural module. - o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = Jasper(input_signal=o_audio_signal, length=o_audio_signal_len) + o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = Jasper( + input_signal=o_audio_signal, length=o_audio_signal_len + ) o_predictions = greedy_decoder(log_probs=o_log_probs) - o_loss = ctc_loss(log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len) + o_loss = ctc_loss( + log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len + ) tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] train_callback = nemo.core.SimpleLossLoggerCallback( tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) ) -#import pdb;pdb.set_trace() +# import pdb;pdb.set_trace() nf.train( tensors_to_optimize=[o_loss], optimizer="novograd", diff --git a/examples/start_here/graph_composition_integration_tests2_0.py b/examples/start_here/graph_composition_integration_tests2_0.py index 317012727741..17be99bcc35d 100644 --- a/examples/start_here/graph_composition_integration_tests2_0.py +++ b/examples/start_here/graph_composition_integration_tests2_0.py @@ -41,7 +41,7 @@ lssg2 = loss(predictions=pg2, target=tg2) -#import pdb;pdb.set_trace() +# import pdb;pdb.set_trace() # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index deb5579dcc16..a980ff3d9802 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -345,14 +345,12 @@ def nest(self, inner_graph, inner_graph_args): # them only for ports that are already indicated as the "bound" ones in the inner graph. self.default_output_binding = False - # Now "copy" steps, i.e. append them to the list, following the original order. - # Moreover, actually execute each step. + # Now "copy" graph execution order and topology by actually executing each step of the nested graph. for _, module_name in inner_graph.steps.items(): - # Add step - store the module name. - self._steps[len(self._steps)] = module_name + # Both module and step will be added by the modules' call(). # Get the module. - module = self._modules[module_name] + module = inner_graph._modules[module_name] # Produce list of arguments that will be passed to a given modules. module_args = {} @@ -383,7 +381,7 @@ def nest(self, inner_graph, inner_graph_args): # Now, the tensor is already produced in outer (i.e. this) graph! module_args[input_port_name] = self.tensors[producer_name][producer_port_name] - #import pdb;pdb.set_trace() + # import pdb;pdb.set_trace() # Ok, now we have all keyword arguments. We can call() the module. # This will collect all the produced output tensors and add them to this graph. module(**module_args) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index a50e3e871672..c0cc34b0594d 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -353,7 +353,7 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) # Update parameters with additional ones. init_params.update(overwrite_params) # TODO: Add section name as default module name! - #if section_name is not None: + # if section_name is not None: # init_params.update({"name": section_name}) # Create and return the object. @@ -530,7 +530,9 @@ def __call__(self, **kwargs): # Create output tensors. output_tensors = [] for out_name, out_type in output_port_defs.items(): - output_tensors.append(NmTensor(producer=self, producer_args=kwargs, output_port_name=out_name, ntype=out_type,)) + output_tensors.append( + NmTensor(producer=self, producer_args=kwargs, output_port_name=out_name, ntype=out_type,) + ) # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). output_class_name = f'{self.__class__.__name__}Output' diff --git a/nemo/utils/bound_outputs.py b/nemo/utils/bound_outputs.py index 86731b763eb2..753e7304925b 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/utils/bound_outputs.py @@ -20,6 +20,7 @@ from nemo.utils import logging + class BoundOutput(object): """ A helper class represenging a single bound output. """ @@ -117,13 +118,16 @@ def bind(self, tensors_list): tensors_list: List of tensors to be added. """ for tensor in tensors_list: - # Use the name being combination of producer and port names. - name = tensor.producer_name + "_" + tensor.name - # Check the presence of the port name in "default" dictionary - this is rather impossible, but... + # Try to use the "default" name = output_port_name. + name = tensor.name + # Check the presence of the port name in "default" dictionary. if name in self._default_outputs.keys(): + # Name present - use the name being combination of producer and port names. + name = tensor.producer_name + "_" + tensor.name + logging.warning( - "Overwriting the already bound output port `{}` produced by `{}`".format( - name, self._default_outputs[name].producer_name + "Setting unigue name of the default output port `{}` produced by `{}` to `{}`".format( + tensor.name, self._default_outputs[tensor.name]._producer_port.module_name, name ) ) # Still, "overwrite" it. diff --git a/tests/unit/core/test_module_initialization.py b/tests/unit/core/test_module_initialization.py index 24071474d2ba..4814cdfadd98 100644 --- a/tests/unit/core/test_module_initialization.py +++ b/tests/unit/core/test_module_initialization.py @@ -76,7 +76,7 @@ def test_call_TaylorNet(self): x_tg = NmTensor( producer=None, producer_args=None, - name=None, + output_port_name=None, ntype=NeuralType(elements_type=ChannelType(), axes=('B', 'D')), ) diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index ee11f53b4186..c4f9c1a9b264 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -111,9 +111,9 @@ def test_graph_nesting_possible_operation_modes(self): @pytest.mark.unit def test_output_ports_binding(self): # Create modules. - data_source = RealFunctionDataLayer(n=100, batch_size=1) - tn = TaylorNet(dim=4) - loss = MSELoss() + data_source = RealFunctionDataLayer(n=100, batch_size=1, name="tgn_ds") + tn = TaylorNet(dim=4, name="tgn_tn") + loss = MSELoss(name="tgn_loss") # Test default binding. with NeuralGraph(operation_mode=OperationMode.training) as g1: @@ -123,10 +123,10 @@ def test_output_ports_binding(self): lss = loss(predictions=y_pred, target=y) assert len(g1.output_ports) == 4 - assert g1.output_ports["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME # Test manual binding. with g1: @@ -134,8 +134,11 @@ def test_output_ports_binding(self): g1.output_ports["my_loss"] = lss assert len(g1.output_ports) == 2 - assert g1.output_ports["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert ( + g1.output_ports.tensors["my_prediction"].compare(tn.output_ports["y_pred"]) + == NeuralTypeComparisonResult.SAME + ) + assert g1.output_ports.tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME @pytest.mark.unit def test_graph_nesting_topology_copy_one_module_defaults(self): @@ -148,7 +151,7 @@ def test_graph_nesting_topology_copy_one_module_defaults(self): with NeuralGraph(operation_mode=OperationMode.training, name="t1_g2") as g2: xg2, tg2 = g1() - + # import pdb;pdb.set_trace() # We expect that both graphs will have the same modes/steps. assert len(g1.steps) == len(g2.steps) assert g1.steps[0] == g2.steps[0] diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/test_neural_graphs.py index dcb15f041f3e..bbc30f3f6d48 100644 --- a/tests/unit/core/test_neural_graphs.py +++ b/tests/unit/core/test_neural_graphs.py @@ -95,6 +95,6 @@ def test_default_output_ports(self): # Tests output ports. assert len(g1.output_ports) == 3 - assert g1.output_ports["x"].compare(x) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["y"].compare(t) == NeuralTypeComparisonResult.SAME - assert g1.output_ports["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["x"].compare(x) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["y"].compare(t) == NeuralTypeComparisonResult.SAME + assert g1.output_ports.tensors["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index 5e1fac351368..e39b0bbbc8e3 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -19,6 +19,7 @@ import pytest from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph from nemo.core.neural_types import NeuralTypeComparisonResult from nemo.utils.bound_outputs import BoundOutputs @@ -32,13 +33,14 @@ def test_binding(self): tn = TaylorNet(dim=4) loss = MSELoss() - # Create the graph by connnecting the modules. - x, y = data_source() - y_pred = tn(x=x) - lss = loss(predictions=y_pred, target=y) + with NeuralGraph() as g: + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = tn(x=x) + lss = loss(predictions=y_pred, target=y) # Test default binding. - bound_outputs = BoundOutputs() + bound_outputs = BoundOutputs(g.tensors) bound_outputs.bind([x, y]) bound_outputs.bind([y_pred]) From cb250dae1ad982cf078b9fd4f40b80f3420256ec Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 21 Apr 2020 15:38:53 -0700 Subject: [PATCH 044/106] refactoring and cleanup of neural graphs Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests3_1.py | 7 +- nemo/core/__init__.py | 3 +- nemo/core/neural_graph/__init__.py | 23 ++++ .../neural_graph/graph_inputs.py} | 6 +- .../neural_graph/graph_outputs.py} | 26 ++--- nemo/core/{ => neural_graph}/neural_graph.py | 100 ++++++++++++------ .../neural_graph_manager.py | 2 +- nemo/core/neural_modules.py | 12 +-- nemo/utils/__init__.py | 2 - tests/unit/utils/test_bound_outputs.py | 6 +- 10 files changed, 118 insertions(+), 69 deletions(-) create mode 100644 nemo/core/neural_graph/__init__.py rename nemo/{utils/bound_inputs.py => core/neural_graph/graph_inputs.py} (97%) rename nemo/{utils/bound_outputs.py => core/neural_graph/graph_outputs.py} (87%) rename nemo/core/{ => neural_graph}/neural_graph.py (88%) rename nemo/core/{ => neural_graph}/neural_graph_manager.py (97%) diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index 944838100e50..aa687ed11cb0 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -36,9 +36,9 @@ with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: # Manually bind input port: "input" -> "x" - g2.input_ports["input"] = fx.input_ports["x"] + g2.inputs["input"] = fx.input_ports["x"] # Add module to graph and bind it input port 'x'. - y = fx(x=g2.input_ports["input"]) + y = fx(x=g2.inputs["input"]) # lss = loss(predictions=y, target=g2.input_ports["input"]) # Build the training graph. @@ -46,9 +46,6 @@ # Add modules to graph. x, t = dl() # Incorporate modules from the existing graph. - import pdb - - pdb.set_trace() p = g2(input=x) lss = loss(predictions=p, target=t) diff --git a/nemo/core/__init__.py b/nemo/core/__init__.py index 56882194c05b..86a3e9c4236e 100644 --- a/nemo/core/__init__.py +++ b/nemo/core/__init__.py @@ -17,7 +17,6 @@ from nemo.core.callbacks import * from nemo.core.neural_factory import * -from nemo.core.neural_graph import NeuralGraph -from nemo.core.neural_graph_manager import NeuralGraphManager +from nemo.core.neural_graph import * from nemo.core.neural_modules import * from nemo.core.neural_types import * diff --git a/nemo/core/neural_graph/__init__.py b/nemo/core/neural_graph/__init__.py new file mode 100644 index 000000000000..bfd0ec33af90 --- /dev/null +++ b/nemo/core/neural_graph/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from nemo.core.neural_graph.neural_graph_manager import NeuralGraphManager +from nemo.core.neural_graph.graph_inputs import GraphInputs, GraphInput +from nemo.core.neural_graph.graph_outputs import GraphOutputs, GraphOutput +from nemo.core.neural_graph.neural_graph import * + diff --git a/nemo/utils/bound_inputs.py b/nemo/core/neural_graph/graph_inputs.py similarity index 97% rename from nemo/utils/bound_inputs.py rename to nemo/core/neural_graph/graph_inputs.py index 7f4e736a1ef7..41b8bd061de2 100644 --- a/nemo/utils/bound_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -26,7 +26,7 @@ # from nemo.core.neural_types import NeuralType -class BoundInput(object): +class GraphInput(object): """ A helper class represenging a single bound input. """ def __init__(self, type): @@ -61,7 +61,7 @@ def consumers_ports(self): return self._consumers -class BoundInputs(MutableMapping): +class GraphInputs(MutableMapping): ''' A specialized dictionary that contains bound inputs of a Neural Graph. ''' @@ -86,7 +86,7 @@ def __setitem__(self, key, value): # Ok, add definition to list of mapped (module, port)s. # Note: for now, there are no mapped modules. - self._inputs[key] = BoundInput(type=value) + self._inputs[key] = GraphInput(type=value) def __getitem__(self, key): """ Returns bound input. """ diff --git a/nemo/utils/bound_outputs.py b/nemo/core/neural_graph/graph_outputs.py similarity index 87% rename from nemo/utils/bound_outputs.py rename to nemo/core/neural_graph/graph_outputs.py index 753e7304925b..194f54a81e61 100644 --- a/nemo/utils/bound_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -21,7 +21,7 @@ from nemo.utils import logging -class BoundOutput(object): +class GraphOutput(object): """ A helper class represenging a single bound output. """ def __init__(self, type, producer_port): @@ -46,21 +46,21 @@ def producer_port(self): return self._producer_port -class BoundOutputs(MutableMapping): +class GraphOutputs(MutableMapping): ''' A specialized dictionary that contains bound outputs of a Neural Graph. - In fact stores two lists of bound tensors: - - "default" output tensors with default keys taken from outputs of modules (might result in + In fact stores two lists of "outputs": + - "default" outputs with default keys taken from outputs of modules (might result in overwriting some keys), and - - "manual" used for specifying the subset of output tensors, each with a new/different key - When accessing the output tensors, it returns the "manual" tensors. If "manual" tensors are not defined, - will return/work on "default" tensors. + - "manual" used for specifying the subset of outputs, each with a new/different key + When accessing the outputs, it returns the "manual" outputs. If "manual" outputs are not defined, + will return/work on "default" outputs. ''' def __init__(self, tensors_list): """ Initializes two (empty) dictionaries. """ - # List + # List of tensors - passed from the external neural graph object. self._tensors_list = tensors_list # This dictionary stores the output tensors collected during the "default" tensor recording. @@ -74,21 +74,21 @@ def __init__(self, tensors_list): def __setitem__(self, key, value): """ - This method is used to set the manual output - creates a BoundOutput item and adds it to the list. + This method is used to set the manual output - creates a GraphOutput item and adds it to the list. Args: key: name of the output (port). - value: tensor that will be used to create BoundOutput. + value: tensor that will be used to create GraphOutput. """ # Make sure that user passed a NmTensor. assert type(value).__name__ == "NmTensor" if key in self._manual_outputs.keys(): raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) # Ok, set output. - self._manual_outputs[key] = BoundOutput(value.type, value.producer_port) + self._manual_outputs[key] = GraphOutput(value.type, value.producer_port) def __getitem__(self, key): - """ Returns BoundOutput - depending whether there are some manual outputs or not. """ + """ Returns GraphOutput - depending whether there are some manual outputs or not. """ if len(self._manual_outputs) > 0: return self._manual_outputs[key] else: # Use default dict. @@ -131,7 +131,7 @@ def bind(self, tensors_list): ) ) # Still, "overwrite" it. - self._default_outputs[name] = BoundOutput(tensor.type, tensor.producer_port) + self._default_outputs[name] = GraphOutput(tensor.type, tensor.producer_port) @property def definitions(self): diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph/neural_graph.py similarity index 88% rename from nemo/core/neural_graph.py rename to nemo/core/neural_graph/neural_graph.py index a980ff3d9802..3ffa4bd12d1a 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -16,14 +16,18 @@ # limitations under the License. # ============================================================================= +__all__ = [ + 'NeuralGraph', +] + from collections import OrderedDict, namedtuple from typing import Dict, Optional from nemo.core import OperationMode from nemo.core.neural_interface import NeuralInterface -from nemo.core.neural_types import NeuralPortNameMismatchError, NmTensor -from nemo.utils.bound_inputs import BoundInput, BoundInputs -from nemo.utils.bound_outputs import BoundOutputs +from nemo.core.neural_types import NeuralType, NeuralPortNameMismatchError, NmTensor +from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs +from nemo.core.neural_graph.graph_outputs import GraphOutputs class NeuralGraph(NeuralInterface): @@ -61,10 +65,10 @@ def __init__(self, operation_mode=OperationMode.both, name=None): self._steps = OrderedDict() # Bound inputs. - self._bound_inputs = BoundInputs() + self._inputs = GraphInputs() # Bound outputs. - self._bound_outputs = BoundOutputs(self._all_tensors) + self._outputs = GraphOutputs(self._all_tensors) # Flag indicating whether the "default" output ports/tensors will be automatically bound. self.default_output_binding = True @@ -106,35 +110,63 @@ def __call__(self, **kwargs): return results @property - def input_ports(self): - """Returns definitions of module input ports. + def inputs(self): + """ + Returns graph inputs. + + Returns: + A graph input. + """ + return self._inputs + + @property + def input_ports(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of graph input ports (dict of Neural Types). .. note:: - This method is NOT returning the dictionary with definitions (like Neural Modules), - but the BoundInputs object. - This was required to enable user to bound inputs with the dict's __setitem__ construct. + This method actually returns a dictionary with definitions (like Neural Modules). + In order to get access to actual graph inputs please call the inputs() method. + + Returns: + A graph input ports definitions. + """ + return self._inputs.definitions + + @property + def outputs(self): + """ + Returns graph outputs. Returns: - A graph bound input ports. + A graph outputs. """ - return self._bound_inputs + return self._outputs @property - def output_ports(self): + def output_ports(self) -> Optional[Dict[str, NeuralType]]: """ - Returns module output ports. + Returns definitions of module output ports (dict of Neural Types). .. note:: - This method is NOT returning the dictionary with definitions (like Neural Modules), - but the BoundOutputs object. - This was required to enable user to override the "default bound outputs" - with the dict's __setitem__ construct. + This method actually returns a dictionary with definitions (like Neural Modules). + In order to get access to actual graph outpus please call the outputs() method. Returns: - A graph bound output ports. + A graph output ports definitions. """ - return self._bound_outputs + return self._outputs.definitions + + @property + def output_tensors(self): + """ + Returns graph output tensors. + + Returns: + A graph output tensors. + """ + return self._outputs.tensors @property def modules(self): @@ -233,7 +265,7 @@ def unused_input_processing(self): # Analogically to NeuralModule, at that point the input can be one of three types: # * NeuralGraph -> bind port using the default name and type. - # * BoundInput -> check definition, if ok bind port. + # * GraphInput -> check definition, if ok bind port. # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. @@ -244,7 +276,7 @@ def unused_input_processing(self): raise ConnectionError("Ports can be bound only by passing the active graph object!") # This case: we are nesting one graph into another and must bind input port of one graph in another! - # So generally we will "copy" the BoundInput object, using the same name. + # So generally we will "copy" the GraphInput object, using the same name. # Copy the port "definition" (i.e. is NeuralType) using the same port name. # This might throw an exception if port with that name was already bound! @@ -256,11 +288,11 @@ def unused_input_processing(self): # Please note that there are no "consumers" here - this is a "pure" binding. - elif type(port_content) is BoundInput: - # Check if BoundInput belongs to the outer graph (i.e. self)! + elif type(port_content) is GraphInput: + # Check if GraphInput belongs to the outer graph (i.e. self)! own_port = False for port in self.input_ports.items(): - if port is BoundInput: + if port is GraphInput: own_port = True break if not own_port: @@ -308,7 +340,7 @@ def unused_input_processing(self): else: raise TypeError( - "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) + "Input '{}' can be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) ) def nest(self, inner_graph, inner_graph_args): @@ -360,7 +392,7 @@ def nest(self, inner_graph, inner_graph_args): # - checking if we have already tensors leading to that input (in outer graph). for input_port_name in module.input_ports.keys(): # Check if this port was bound in the inner graph. - key = inner_graph.input_ports.has_binding(module_name, input_port_name) + key = inner_graph.inputs.has_binding(module_name, input_port_name) # If so, then we must pass whatever was passed to that port in the list of arguments. if key is not None: module_args[input_port_name] = inner_graph_args[key] @@ -391,12 +423,12 @@ def nest(self, inner_graph, inner_graph_args): # This part is different from Neural Module. # Now the goal is NOT to create NEW "tensors", but to return the BOUND ones! - # Still, those must be bound in the outer (active) graph. + # Still, those must be bound in the outer (active) graph, but using port names from the inner (nested) graph. # Get list of "the adequate output tensors". output_tensors = {} # Iterate through outputs of the inner graph. - for key, tensor in inner_graph._bound_outputs.tensors.items(): + for key, tensor in inner_graph.output_tensors.items(): # Find the tensors within this (outer) graph that are outpus by the same producer-port. producer_name = tensor.producer_name producer_port_name = tensor.name @@ -409,7 +441,7 @@ def nest(self, inner_graph, inner_graph_args): # Bind the "default" output ports of the inner graph as "default" output ports of this graph. # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. - self._bound_outputs.bind([results]) + self.outputs.bind([results]) else: # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). @@ -421,7 +453,7 @@ def nest(self, inner_graph, inner_graph_args): # Bind the "default" output ports of the inner graph as "default" output ports of this graph. # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. - self._bound_outputs.bind(output_tensors.values()) + self.outputs.bind(output_tensors.values()) # Ok, now we can turn automatic binding on. self.default_output_binding = True @@ -448,14 +480,14 @@ def bind_outputs(self, tensors_list): # Bind the tensors as graph outputs. if self.default_output_binding: - self._bound_outputs.bind(tensors_list) + self.outputs.bind(tensors_list) @property def tensors(self): """ Returns the dictionary of all output tensors, aggregated by modules (key). """ return self._all_tensors - def show_bound_inputs(self): + def show_inputs(self): print("bound input ports: ") # for key, value in self._bound_input_ports.items(): # print(" * `{}`: `{}` ({})".format(key, value, type(value))) @@ -464,7 +496,7 @@ def show_bound_inputs(self): # for key, value in self._bound_input_tensors.items(): # print(" * `{}`: `{}` ({})".format(key, value, type(value))) - def show_bound_outputs(self): + def show_outputs(self): print("bound (default) output ports: ") # for key, value in self._bound_output_ports_default.items(): # print(" * `{}`: `{}` ({})".format(key, value, type(value))) diff --git a/nemo/core/neural_graph_manager.py b/nemo/core/neural_graph/neural_graph_manager.py similarity index 97% rename from nemo/core/neural_graph_manager.py rename to nemo/core/neural_graph/neural_graph_manager.py index 8fc52397b7c7..edbca32eb4d5 100644 --- a/nemo/core/neural_graph_manager.py +++ b/nemo/core/neural_graph/neural_graph_manager.py @@ -17,7 +17,7 @@ # ============================================================================= from nemo.core.neural_factory import OperationMode -from nemo.core.neural_graph import NeuralGraph +from nemo.core.neural_graph.neural_graph import NeuralGraph from nemo.utils.object_registry import ObjectRegistry diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index c0cc34b0594d..6acb6d9d78c0 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -33,7 +33,7 @@ from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.bound_inputs import BoundInput +from nemo.core.neural_graph.graph_inputs import GraphInput from nemo.utils.decorators.deprecated import deprecated from nemo.utils.module_port import ModulePort @@ -469,7 +469,7 @@ def __call__(self, **kwargs): # Ok, at that point the input can be one of three types: # * NeuralGraph -> bind port using the default name and type. - # * BoundInput -> check definition, if ok bind port + # * GraphInput -> check definition, if ok bind port # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. @@ -481,15 +481,15 @@ def __call__(self, **kwargs): active_graph = port_content # Copy the port "definition" (i.e. is NeuralType) using the same port name. - active_graph.input_ports[port_name] = input_port_defs[port_name] + active_graph.inputs[port_name] = input_port_defs[port_name] # Bind the neural graph input port, i.e. remember that a given graph port should pass data # to THIS module (when it finally will be connected). - active_graph.input_ports[port_name].bind([ModulePort(self.name, port_name)]) + active_graph.inputs[port_name].bind([ModulePort(self.name, port_name)]) # Please note that there are no "consumers" here - this is a "pure" binding. - elif type(port_content) is BoundInput: + elif type(port_content) is GraphInput: # Compare input port definition with the received definition. input_port_defs[port_name].compare_and_raise_error( @@ -511,7 +511,7 @@ def __call__(self, **kwargs): else: raise TypeError( - "Input '{}' can be of one of three types: NeuralGraph, BoundInput or NmTensor".format(port_name) + "Input '{}' can be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) ) ###### PRODUCE OUTPUTS. ###### diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 437cbc1df88b..b5cca20046ac 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -26,5 +26,3 @@ from nemo.utils.app_state import AppState from nemo.utils.object_registry import ObjectRegistry from nemo.utils.module_port import ModulePort -from nemo.utils.bound_inputs import BoundInputs, BoundInput -from nemo.utils.bound_outputs import BoundOutputs diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/utils/test_bound_outputs.py index e39b0bbbc8e3..77882aa9bda8 100644 --- a/tests/unit/utils/test_bound_outputs.py +++ b/tests/unit/utils/test_bound_outputs.py @@ -21,11 +21,11 @@ from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import NeuralGraph from nemo.core.neural_types import NeuralTypeComparisonResult -from nemo.utils.bound_outputs import BoundOutputs +from nemo.core.neural_graph.graph_outputs import GraphOutputs @pytest.mark.usefixtures("neural_factory") -class TestBoundOutputs: +class TestGraphOutputs: @pytest.mark.unit def test_binding(self): # Create modules. @@ -40,7 +40,7 @@ def test_binding(self): lss = loss(predictions=y_pred, target=y) # Test default binding. - bound_outputs = BoundOutputs(g.tensors) + bound_outputs = GraphOutputs(g.tensors) bound_outputs.bind([x, y]) bound_outputs.bind([y_pred]) From 6907cd72b856b3393c7b793a946f8839baa7174e Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 21 Apr 2020 16:10:40 -0700 Subject: [PATCH 045/106] Refactoring, cleanups of NeuralGraphs nesting/binding Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 10 +- nemo/core/neural_graph/neural_graph.py | 95 +------------------ nemo/core/neural_modules.py | 45 +++++---- .../test_graph_outputs.py} | 0 tests/unit/core/test_neural_graph_nesting.py | 20 ++-- 5 files changed, 45 insertions(+), 125 deletions(-) rename tests/unit/{utils/test_bound_outputs.py => core/test_graph_outputs.py} (100%) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 41b8bd061de2..20de8c0d3171 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -41,14 +41,14 @@ def __init__(self, type): # List of ModulePort tuples to which this input links to (module name, port name). self._consumers = [] - def bind(self, modules): - """ Binds the modules to this "graph input". + def bind(self, module_ports): + """ Binds the (modules-ports) to this "graph input". Args: - modules: List of ModulePort tuples to be added. + module_ports: List of ModulePort tuples to be added. """ - for module in modules: - self._consumers.append(module) + for module_port in module_ports: + self._consumers.append(module_port) @property def type(self): diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 3ffa4bd12d1a..42de4f299eb8 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -242,8 +242,8 @@ def record_step(self, module): """ Records the operation (module plus passed inputs) on a list. """ - # Check if module with that name already exists. - # TODO: Uncomment when we will refactor all examples so training/validation graphs won't be added + # Check if module with that name already exists - to avoid the potential loops (DAG). + # TODO: Uncomment after we will refactor all the examples, so training/validation graphs won't be added # to the "default" graph. # if module.name in self._modules.keys(): # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) @@ -253,95 +253,6 @@ def record_step(self, module): # Add step - store the module name. self._steps[len(self._steps)] = module.name - def unused_input_processing(self): - """ - Old implementation! To be deleted!!!! Wrrrrrr!!! - """ - # Check inputs: iterate through all inputs passed to the inner graph. - for port_name, port_content in inner_graph_args.items(): - # Make sure that passed arguments correspond to input port names. - if port_name not in inner_graph.input_ports.keys(): - raise NeuralPortNameMismatchError(port_name) - - # Analogically to NeuralModule, at that point the input can be one of three types: - # * NeuralGraph -> bind port using the default name and type. - # * GraphInput -> check definition, if ok bind port. - # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). - - # Check what was actually passed. - if type(port_content) is NeuralGraph: - - # Make sure that port_content is the currently active graph, i.e. THIS GRAPH! - if port_content is not self: - raise ConnectionError("Ports can be bound only by passing the active graph object!") - - # This case: we are nesting one graph into another and must bind input port of one graph in another! - # So generally we will "copy" the GraphInput object, using the same name. - - # Copy the port "definition" (i.e. is NeuralType) using the same port name. - # This might throw an exception if port with that name was already bound! - self.input_ports[port_name] = inner_graph.input_ports[port_name].type - - # Remember that a given graph port should pass data to all "bound modules" of the "nested" graph - # (when it finally will be connected). - self.input_ports[port_name].bind(inner_graph.input_ports[port_name].modules) - - # Please note that there are no "consumers" here - this is a "pure" binding. - - elif type(port_content) is GraphInput: - # Check if GraphInput belongs to the outer graph (i.e. self)! - own_port = False - for port in self.input_ports.items(): - if port is GraphInput: - own_port = True - break - if not own_port: - raise NeuralPortNameMismatchError(port_name) - - # Compare input port definition with the received definition. - port_content.type.compare_and_raise_error( - self.__class__.__name__, port_name, inner_graph.input_ports[port_name].type - ) - - # Remember that a given graph port should pass data to all "bound modules" of the "nested" graph - # (when it finally will be connected). - port_content[port_name].bind(inner_graph.input_ports[port_name].modules) - - # Please note that there are no "consumers" here - this is a "pure" binding. - - elif type(port_content) is NmTensor: - - # Compare input port definition onf the inner graph with the received definition. - inner_graph.input_ports[port_name].type.compare_and_raise_error( - self.__class__.__name__, port_name, port_content.type - ) - # Note that this tensor is already! a part of this graph. - # (It has to be output of one of the previous modules.) - # So we can find it in: self._tensors - - # Reaching that point means that we accepted input to a bound graph port. - # Need to connect it - "copy" all modules connected to this "bound input" as consumers. - for consumer in inner_graph.input_ports[port_name].modules: - # Add consumer. - port_content.add_consumer(consumer.module_name, consumer.port_name) - - # The current graph parsing requires us to update all outputs of - # a module that "accepted" the input. - # In other words: for every "consumer" we need to update all tensors it produced. - # Update means changing the original producer_args for ALL TENSORS IN THE GRAPH produced by - # this module. - # producer_name = consumer.module_name - # if producer_name in self._tensors.keys(): - # # Get all tensor producer by this module. - # for output_tensor in self._tensors[producer_name]: - # # Set "input port value" to new content - which indicates tensor (and producer) - # # that will be used during graph backward traverse. - # output_tensor.producer_args[port_name] = port_content # i.e. Tensor. - - else: - raise TypeError( - "Input '{}' can be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) - ) def nest(self, inner_graph, inner_graph_args): """ @@ -351,7 +262,7 @@ def nest(self, inner_graph, inner_graph_args): inner_graph: Graph to be copied (will be "nested" in this (self) graph). inner_graph_args: inputs passed to the graph call. """ - # "Copy" modules. + # "Copy" the modules from nested graph. for key, module in inner_graph.modules.items(): # Check if module with that name already exists. # TODO: Uncomment when we will refactor all examples so training/validation graphs won't be added diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 6acb6d9d78c0..8556c0aae3a1 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -452,10 +452,7 @@ def __call__(self, **kwargs): # Set the operation mode of the outer graph. self.operation_mode = self._app_state.active_graph.operation_mode - - # Get input and output ports definitions - potentially depending on the operation mode! - input_port_defs = self.input_ports - output_port_defs = self.output_ports + # The input and output ports definitions can potentially depend on the operation mode! # Record the operation (i.e. add a single module). self._app_state.active_graph.record_step(self) @@ -464,12 +461,12 @@ def __call__(self, **kwargs): # Iterate through all passed parameters. for port_name, port_content in kwargs.items(): # Make sure that passed arguments corresponds to one of the input port names. - if port_name not in input_port_defs.keys(): + if port_name not in self.input_ports.keys(): raise NeuralPortNameMismatchError(port_name) - # Ok, at that point the input can be one of three types: + # At that point the input can be one of three types: # * NeuralGraph -> bind port using the default name and type. - # * GraphInput -> check definition, if ok bind port + # * GraphInput -> check definition, if ok bind port. # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. @@ -480,41 +477,53 @@ def __call__(self, **kwargs): # Create an alias so the logic will be more clear. active_graph = port_content - # Copy the port "definition" (i.e. is NeuralType) using the same port name. - active_graph.inputs[port_name] = input_port_defs[port_name] + # This case: we are nesting one graph into another and must bind input port of one graph in another! + # So generally we must "copy" the of thus module to graog (the inverted logic!). + + # Copy the port "definition" (i.e. is NeuralType) using the same port name. + active_graph.inputs[port_name] = self.input_ports[port_name] # Bind the neural graph input port, i.e. remember that a given graph port should pass data - # to THIS module (when it finally will be connected). + # to THIS module-port (when it finally will be connected). active_graph.inputs[port_name].bind([ModulePort(self.name, port_name)]) - # Please note that there are no "consumers" here - this is a "pure" binding. + # Please note that there are no "consumers" here - this is a "pure binding". elif type(port_content) is GraphInput: + # Check if GraphInput belongs to the active graph ! + own_port = False + for port in self._app_state.active_graph.inputs.items(): + if port is GraphInput: + own_port = True + break + if not own_port: + raise NeuralPortNameMismatchError(port_name) + # Compare input port definition with the received definition. - input_port_defs[port_name].compare_and_raise_error( + self.input_ports[port_name].compare_and_raise_error( self.__class__.__name__, port_name, port_content.type ) # Bind the neural graph input port, i.e. remember that a given graph port should pass data - # to THIS module (when it finally will be connected). + # to THIS module-port (when it finally will be connected). port_content.bind([ModulePort(self.name, port_name)]) - # Please note that there are no "consumers" here - this is a "pure" binding. + # Please note that there are no "consumers" here - this is a "pure binding". elif type(port_content) is NmTensor: # Compare input port definition with the received definition. - input_port_defs[port_name].compare_and_raise_error(self.__class__.__name__, port_name, port_content) + self.input_ports[port_name].compare_and_raise_error(self.__class__.__name__, port_name, port_content) - # Ok, can connect, add self (module) as "consumer" to the input tensor. + # Ok, the goal here is to actually "connect": add self (module) as "consumer" to the input tensor. port_content.add_consumer(self.name, port_name) - else: raise TypeError( - "Input '{}' can be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) + "Input '{}' must be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) ) ###### PRODUCE OUTPUTS. ###### + output_port_defs = self.output_ports # Create output tensors. if len(output_port_defs) == 1: # Get port name and type. diff --git a/tests/unit/utils/test_bound_outputs.py b/tests/unit/core/test_graph_outputs.py similarity index 100% rename from tests/unit/utils/test_bound_outputs.py rename to tests/unit/core/test_graph_outputs.py diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index c4f9c1a9b264..613cabaca30b 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -122,23 +122,23 @@ def test_output_ports_binding(self): y_pred = tn(x=x) lss = loss(predictions=y_pred, target=y) - assert len(g1.output_ports) == 4 - assert g1.output_ports.tensors["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports.tensors["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports.tensors["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert g1.output_ports.tensors["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert len(g1.outputs) == 4 + assert g1.output_tensors["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g1.output_tensors["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert g1.output_tensors["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_tensors["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME # Test manual binding. with g1: - g1.output_ports["my_prediction"] = y_pred - g1.output_ports["my_loss"] = lss + g1.outputs["my_prediction"] = y_pred + g1.outputs["my_loss"] = lss - assert len(g1.output_ports) == 2 + assert len(g1.outputs) == 2 assert ( - g1.output_ports.tensors["my_prediction"].compare(tn.output_ports["y_pred"]) + g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME ) - assert g1.output_ports.tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert g1.output_tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME @pytest.mark.unit def test_graph_nesting_topology_copy_one_module_defaults(self): From cb69367269102e1bd8dfd031e102e7b5e96fa11f Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 21 Apr 2020 16:11:05 -0700 Subject: [PATCH 046/106] reformating fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/__init__.py | 7 +++---- nemo/core/neural_graph/neural_graph.py | 5 ++--- nemo/core/neural_modules.py | 2 +- tests/unit/core/test_graph_outputs.py | 2 +- tests/unit/core/test_neural_graph_nesting.py | 5 +---- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/nemo/core/neural_graph/__init__.py b/nemo/core/neural_graph/__init__.py index bfd0ec33af90..4596cb39c10f 100644 --- a/nemo/core/neural_graph/__init__.py +++ b/nemo/core/neural_graph/__init__.py @@ -16,8 +16,7 @@ # limitations under the License. # ============================================================================= -from nemo.core.neural_graph.neural_graph_manager import NeuralGraphManager -from nemo.core.neural_graph.graph_inputs import GraphInputs, GraphInput -from nemo.core.neural_graph.graph_outputs import GraphOutputs, GraphOutput +from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs +from nemo.core.neural_graph.graph_outputs import GraphOutput, GraphOutputs from nemo.core.neural_graph.neural_graph import * - +from nemo.core.neural_graph.neural_graph_manager import NeuralGraphManager diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 42de4f299eb8..fcc5dead8986 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -24,10 +24,10 @@ from typing import Dict, Optional from nemo.core import OperationMode -from nemo.core.neural_interface import NeuralInterface -from nemo.core.neural_types import NeuralType, NeuralPortNameMismatchError, NmTensor from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs +from nemo.core.neural_interface import NeuralInterface +from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor class NeuralGraph(NeuralInterface): @@ -253,7 +253,6 @@ def record_step(self, module): # Add step - store the module name. self._steps[len(self._steps)] = module.name - def nest(self, inner_graph, inner_graph_args): """ Method nests (copies) a graph: modules, steps, topology (tensors). diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 8556c0aae3a1..cbcd0dcaea47 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -29,11 +29,11 @@ from ruamel.yaml import YAML from nemo.core import NeuralGraph, NeuralModuleFactory, OperationMode +from nemo.core.neural_graph.graph_inputs import GraphInput from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.core.neural_graph.graph_inputs import GraphInput from nemo.utils.decorators.deprecated import deprecated from nemo.utils.module_port import ModulePort diff --git a/tests/unit/core/test_graph_outputs.py b/tests/unit/core/test_graph_outputs.py index 77882aa9bda8..df99636d3e39 100644 --- a/tests/unit/core/test_graph_outputs.py +++ b/tests/unit/core/test_graph_outputs.py @@ -20,8 +20,8 @@ from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import NeuralGraph -from nemo.core.neural_types import NeuralTypeComparisonResult from nemo.core.neural_graph.graph_outputs import GraphOutputs +from nemo.core.neural_types import NeuralTypeComparisonResult @pytest.mark.usefixtures("neural_factory") diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 613cabaca30b..ae9cddba7554 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -134,10 +134,7 @@ def test_output_ports_binding(self): g1.outputs["my_loss"] = lss assert len(g1.outputs) == 2 - assert ( - g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) - == NeuralTypeComparisonResult.SAME - ) + assert g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert g1.output_tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME @pytest.mark.unit From 8389f211e3517c26cb78c92b86c6b66999a3098b Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 21 Apr 2020 18:15:57 -0700 Subject: [PATCH 047/106] Fixes, unit tests for binding of nested graphs, automatic and manual, inputs and outputs: passing Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 14 +- nemo/core/neural_modules.py | 4 +- ...tputs.py => test_graph_outputs_binding.py} | 44 +++- tests/unit/core/test_neural_graph_nesting.py | 208 ++++++++++++++---- 4 files changed, 223 insertions(+), 47 deletions(-) rename tests/unit/core/{test_graph_outputs.py => test_graph_outputs_binding.py} (60%) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 20de8c0d3171..74d6271be8f4 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -19,6 +19,7 @@ from collections import namedtuple from collections.abc import MutableMapping +from nemo.core.neural_types import NeuralType from nemo.utils import logging from nemo.utils.module_port import ModulePort @@ -80,13 +81,16 @@ def __setitem__(self, key, value): """ if key in self._inputs.keys(): raise KeyError("Overwriting definition of a previously bound port `{}` is not allowed".format(key)) - # Make sure that a proper NeuralType definition was passed here. - # if type(value) is not NeuralType: - # raise TypeError("Port `{}` definition must be must be a NeuralType".format(key)) + # Make sure that a proper NeuralType definition was passed here. + if isinstance(value, NeuralType): + val_type = value + elif isinstance(value, GraphInput): + val_type = value.type + raise TypeError("Port `{}` definition must be must be a NeuralType or GraphInput type".format(key)) # Ok, add definition to list of mapped (module, port)s. - # Note: for now, there are no mapped modules. - self._inputs[key] = GraphInput(type=value) + # Note: for now, there are no mapped modules, so copy only (neural) type. + self._inputs[key] = GraphInput(type=val_type) def __getitem__(self, key): """ Returns bound input. """ diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index cbcd0dcaea47..c3eea575e34c 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -493,8 +493,8 @@ def __call__(self, **kwargs): # Check if GraphInput belongs to the active graph ! own_port = False - for port in self._app_state.active_graph.inputs.items(): - if port is GraphInput: + for gcontent in self._app_state.active_graph.inputs.values(): + if gcontent is port_content: own_port = True break if not own_port: diff --git a/tests/unit/core/test_graph_outputs.py b/tests/unit/core/test_graph_outputs_binding.py similarity index 60% rename from tests/unit/core/test_graph_outputs.py rename to tests/unit/core/test_graph_outputs_binding.py index df99636d3e39..7ee2678a6de9 100644 --- a/tests/unit/core/test_graph_outputs.py +++ b/tests/unit/core/test_graph_outputs_binding.py @@ -19,7 +19,7 @@ import pytest from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import NeuralGraph +from nemo.core import NeuralGraph, OperationMode from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_types import NeuralTypeComparisonResult @@ -27,7 +27,7 @@ @pytest.mark.usefixtures("neural_factory") class TestGraphOutputs: @pytest.mark.unit - def test_binding(self): + def test_graph_outputs1_binding(self): # Create modules. data_source = RealFunctionDataLayer(n=100, batch_size=1) tn = TaylorNet(dim=4) @@ -76,3 +76,43 @@ def test_binding(self): with pytest.raises(KeyError): _ = defs["x"] + + @pytest.mark.unit + def test_graph_outputs2_binding(self): + # Create modules. + data_source = RealFunctionDataLayer(n=100, batch_size=1, name="tgo2_ds") + tn = TaylorNet(dim=4, name="tgo2_tn") + loss = MSELoss(name="tgo2_loss") + + # Test default binding. + with NeuralGraph(operation_mode=OperationMode.training) as g1: + # Create the graph by connnecting the modules. + x, y = data_source() + y_pred = tn(x=x) + lss = loss(predictions=y_pred, target=y) + + assert len(g1.outputs) == 4 + # Test ports. + for (module, port, tensor) in [ + (data_source, "x", x), + (data_source, "y", y), + (tn, "y_pred", y_pred), + (loss, "loss", lss), + ]: + # Compare definitions - from outputs. + assert g1.outputs[port].type.compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME + # Compare definitions - from output_ports. + assert g1.output_ports[port].compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME + # Compare definitions - from output_tensors. + assert g1.output_tensors[port].compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME + # Make sure that tensor was bound, i.e. iput refers to the same object instance! + assert g1.output_tensors[port] is tensor + + # Test manual binding. + with g1: + g1.outputs["my_prediction"] = y_pred + g1.outputs["my_loss"] = lss + + assert len(g1.outputs) == 2 + assert g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index ae9cddba7554..a5fe16bcb5ee 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -30,7 +30,7 @@ @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphNesting: @pytest.mark.unit - def test_module_nesting_change_operation_modes(self): + def test_module_nesting1_change_operation_modes(self): """ Tests whether invalid nesting (i.e. nesting of graphs with incompatible modes) throw exeptions. """ @@ -50,12 +50,12 @@ def test_module_nesting_change_operation_modes(self): assert dl.operation_mode == OperationMode.inference @pytest.mark.unit - def test_graph_nesting_possible_operation_modes(self): + def test_graph_nesting2_possible_operation_modes(self): """ Tests whether invalid nesting (i.e. nesting of graphs with incompatible modes) throw exeptions. """ # Instantiate the necessary neural modules. - dl = RealFunctionDataLayer(n=100, batch_size=4) + dl = RealFunctionDataLayer(n=10, batch_size=1) with NeuralGraph(operation_mode=OperationMode.both) as both: _, _ = dl() @@ -109,48 +109,180 @@ def test_graph_nesting_possible_operation_modes(self): _, _ = inference() @pytest.mark.unit - def test_output_ports_binding(self): - # Create modules. - data_source = RealFunctionDataLayer(n=100, batch_size=1, name="tgn_ds") - tn = TaylorNet(dim=4, name="tgn_tn") - loss = MSELoss(name="tgn_loss") + def test_graph_nesting3_topology_copy_one_module_default_outputs(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: binding of outputs, default port names. + """ + dl = RealFunctionDataLayer(n=10, batch_size=1, name="tgn3_dl") - # Test default binding. - with NeuralGraph(operation_mode=OperationMode.training) as g1: - # Create the graph by connnecting the modules. - x, y = data_source() - y_pred = tn(x=x) - lss = loss(predictions=y_pred, target=y) - - assert len(g1.outputs) == 4 - assert g1.output_tensors["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME - assert g1.output_tensors["y"].compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME - assert g1.output_tensors["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert g1.output_tensors["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - - # Test manual binding. - with g1: - g1.outputs["my_prediction"] = y_pred - g1.outputs["my_loss"] = lss - - assert len(g1.outputs) == 2 - assert g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert g1.output_tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn3_g1") as g1: + xg1, tg1 = dl() + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn3_g2") as g2: + xg2, tg2 = g1() + + # We expect that both graphs will have the same steps. + assert len(g1.steps) == len(g2.steps) + assert g1.steps[0] == g2.steps[0] + + # Make sure that the modules are the same. + assert len(g1) == len(g2) + assert g1["tgn3_dl"] is dl + assert g2["tgn3_dl"] is dl + assert g1["tgn3_dl"] is g2["tgn3_dl"] + + # Make sure that outputs are ok. + assert len(g1.outputs) == len(g2.outputs) + for port in ["x", "y"]: + # Definitions are the same: test two "paths" of accessing the type. + assert g1.outputs[port].type.compare(g1.output_ports[port]) == NeuralTypeComparisonResult.SAME + + assert g1.output_ports[port].compare(g2.output_ports[port]) == NeuralTypeComparisonResult.SAME + assert g1.outputs[port].type.compare(g2.outputs[port].type) == NeuralTypeComparisonResult.SAME + # At the same time - those have to be two different port objects! + assert g1.outputs[port] is not g2.outputs[port] + # And different tensors (as those are "internally produced tensors"!) + assert g1.output_tensors[port] is not g2.output_tensors[port] @pytest.mark.unit - def test_graph_nesting_topology_copy_one_module_defaults(self): - """ Test whether when nesting one graph into another the graph topology (tensors) will be copied. """ + def test_graph_nesting4_topology_copy_one_module_manual_outputs(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: binding of outputs, manual port names. + """ - dl = RealFunctionDataLayer(n=100, batch_size=32, name="t1_dl") + dl = RealFunctionDataLayer(n=10, batch_size=1, name="tgn4_dl") - with NeuralGraph(operation_mode=OperationMode.training, name="t1_g1") as g1: + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn4_g1") as g1: xg1, tg1 = dl() + # Set port binding manually, with different names - and their number! + g1.outputs["inner_x"] = xg1 - with NeuralGraph(operation_mode=OperationMode.training, name="t1_g2") as g2: - xg2, tg2 = g1() - # import pdb;pdb.set_trace() - # We expect that both graphs will have the same modes/steps. + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn4_g2") as g2: + xg2 = g1() + # Set port binding manually, with different names - and their number! + g2.outputs["outer_x"] = xg2 + + # We expect that both graphs will have the same steps. assert len(g1.steps) == len(g2.steps) assert g1.steps[0] == g2.steps[0] + + # Make sure that the modules are the same. assert len(g1) == len(g2) - assert g1["t1_dl"] is g2["t1_dl"] + assert g1["tgn4_dl"] is g2["tgn4_dl"] + + # Make sure that outputs are ok. + assert len(g1.outputs) == len(g2.outputs) + for inter_port, outer_port in [("inner_x", "outer_x")]: + # Definitions are the same: test two "paths" of accessing the type. + assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME + assert g1.outputs[inter_port].type.compare(g2.outputs[outer_port].type) == NeuralTypeComparisonResult.SAME + # At the same time - those have to be two different port objects! + assert g1.outputs[inter_port] is not g2.outputs[outer_port] + # And different tensors (as those are "internally produced tensors"!) + assert g1.output_tensors[inter_port] is not g2.output_tensors[outer_port] + + @pytest.mark.unit + def test_graph_nesting5_topology_copy_one_module_default_inputs(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: binding of inputs, default port names. + """ + tn = TaylorNet(dim=4, name="tgn5_tn") + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training) as g1: + y_pred1 = tn(x=g1) + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training) as g2: + y_pred2 = g1(x=g2) + + # We expect that both graphs will have the same steps. + assert len(g1.steps) == len(g2.steps) + assert g1.steps[0] == g2.steps[0] + + # Make sure that the modules are the same. + assert len(g1) == len(g2) + assert g1["tgn5_tn"] is g2["tgn5_tn"] + + # Make sure that inputs are ok. + assert len(g1.inputs) == len(g2.inputs) + assert g1.input_ports["x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g2.input_ports["x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + # At the same time - those point to the same module-port. + assert g1.inputs.has_binding(tn.name, "x") + assert g2.inputs.has_binding(tn.name, "x") + assert g1.inputs["x"].consumers_ports[0].module_name == tn.name + assert g1.inputs["x"].consumers_ports[0].port_name == "x" + assert g2.inputs["x"].consumers_ports[0].module_name == tn.name + assert g2.inputs["x"].consumers_ports[0].port_name == "x" + + # Make sure that outputs are ok. + assert len(g1.outputs) == len(g2.outputs) + assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + # At the same time - those have to be two different port objects! + assert g1.outputs["y_pred"] is not g2.outputs["y_pred"] + # And different tensors (as those are "internally produced tensors"!) + assert g1.output_tensors["y_pred"] is y_pred1 + assert g2.output_tensors["y_pred"] is y_pred2 + assert y_pred1 is not y_pred2 + + @pytest.mark.unit + def test_graph_nesting6_topology_copy_one_module_manual_inputs(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: binding of inputs, manual port names. + """ + tn = TaylorNet(dim=4, name="tgn6_tn") + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn6_g1") as g1: + # Copy input type. + g1.inputs["inner_x"] = tn.input_ports["x"] + # Bind the input port. + y_pred1 = tn(x=g1.inputs["inner_x"]) + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn6_g2") as g2: + # Copy input type. + g2.inputs["outer_x"] = g1.input_ports["inner_x"] + # Bind the input port. + y_pred2 = g1(inner_x=g2.inputs["outer_x"]) + + # We expect that both graphs will have the same steps. + assert len(g1.steps) == len(g2.steps) + assert g1.steps[0] == g2.steps[0] + + # Make sure that the modules are the same. + assert len(g1) == len(g2) + assert g1["tgn6_tn"] is g2["tgn6_tn"] + + # Make sure that inputs are ok. + assert len(g1.inputs) == len(g2.inputs) + assert g1.input_ports["inner_x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g2.input_ports["outer_x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + # At the same time - those point to the same module-port. + assert g1.inputs.has_binding(tn.name, "x") + assert g2.inputs.has_binding(tn.name, "x") + assert g1.inputs["inner_x"].consumers_ports[0].module_name == tn.name + assert g1.inputs["inner_x"].consumers_ports[0].port_name == "x" + assert g2.inputs["outer_x"].consumers_ports[0].module_name == tn.name + assert g2.inputs["outer_x"].consumers_ports[0].port_name == "x" + + # Make sure that outputs are ok. + assert len(g1.outputs) == len(g2.outputs) + assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + # At the same time - those have to be two different port objects! + assert g1.outputs["y_pred"] is not g2.outputs["y_pred"] + # And different tensors (as those are "internally produced tensors"!) + assert g1.output_tensors["y_pred"] is y_pred1 + assert g2.output_tensors["y_pred"] is y_pred2 + assert y_pred1 is not y_pred2 From 9235ca88c47f9cd354d360392ae5fb197cfbc4af Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 12:46:58 -0700 Subject: [PATCH 048/106] Added several tests for graph nesting, covered (and fixed) case of passing names from inner to outer graph in manual output port binding Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 11 +- nemo/core/neural_graph/neural_graph.py | 9 +- tests/unit/core/test_neural_graph_nesting.py | 115 ++++++++++++++++++- tests/unit/core/test_neural_graphs.py | 12 +- 4 files changed, 133 insertions(+), 14 deletions(-) diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 194f54a81e61..1d3cf1ad9527 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -111,15 +111,18 @@ def __len__(self): else: # Use default dict. return len(self._default_outputs) - def bind(self, tensors_list): + def bind(self, tensors_list, port_names = None): """ Binds the default outputs. Args: tensors_list: List of tensors to be added. + port_names: List of port names (visible outside). If None: using internal tensor "output port names". """ - for tensor in tensors_list: - # Try to use the "default" name = output_port_name. - name = tensor.name + # Set names. + if port_names is None: + port_names = [tensor.name for tensor in tensors_list] + + for name, tensor in zip(port_names, tensors_list): # Check the presence of the port name in "default" dictionary. if name in self._default_outputs.keys(): # Name present - use the name being combination of producer and port names. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index fcc5dead8986..1cc4c05beef9 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -347,11 +347,13 @@ def nest(self, inner_graph, inner_graph_args): if len(output_tensors) == 1: # Return a single tensor. - results = next(iter(output_tensors.values())) + key = list(output_tensors)[0] + results = output_tensors[key] # Bind the "default" output ports of the inner graph as "default" output ports of this graph. # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. - self.outputs.bind([results]) + # But: Use output port name of the inner graph! + self.outputs.bind([results], [key]) else: # Create a named tuple type enabling to access outputs by attributes (e.g. out.x). @@ -363,7 +365,8 @@ def nest(self, inner_graph, inner_graph_args): # Bind the "default" output ports of the inner graph as "default" output ports of this graph. # Call the bind() method of bound_outputs directly, as we already have the tensors in our graph. - self.outputs.bind(output_tensors.values()) + # But: Use output port name of the inner graph! + self.outputs.bind(output_tensors.values(), output_tensors.keys()) # Ok, now we can turn automatic binding on. self.default_output_binding = True diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index a5fe16bcb5ee..78c0855564cb 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -35,7 +35,7 @@ def test_module_nesting1_change_operation_modes(self): Tests whether invalid nesting (i.e. nesting of graphs with incompatible modes) throw exeptions. """ # Instantiate the necessary neural modules. - dl = RealFunctionDataLayer(n=100, batch_size=4) + dl = RealFunctionDataLayer(n=10, batch_size=1) with NeuralGraph(operation_mode=OperationMode.both): _, _ = dl() @@ -187,6 +187,47 @@ def test_graph_nesting4_topology_copy_one_module_manual_outputs(self): # And different tensors (as those are "internally produced tensors"!) assert g1.output_tensors[inter_port] is not g2.output_tensors[outer_port] + + @pytest.mark.unit + def test_graph_nesting4_1_topology_copy_one_module_manual_outputs_bound_only_in_inner(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: binding of outputs, manual port names - only in the inner graph. + Testing whether outputs of outer graph have the manually bound names. + """ + + dl = RealFunctionDataLayer(n=10, batch_size=1, name="tgn41_dl") + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn41_g1") as g1: + xg1, tg1 = dl() + # Set port binding manually, with different names - and their number! + g1.outputs["inner_x"] = xg1 + g1.outputs["inner_t"] = tg1 + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn41_g2") as g2: + # Get them as a tuple. + outputs = g1() + + # Retrieve tensors from tuple. + assert outputs._fields[0] == "inner_x" + assert outputs._fields[1] == "inner_t" + xg2 = outputs.inner_x + tg2 = outputs.inner_t + + # Make sure that outer graph has objects of the same names + assert len(g1.outputs) == len(g2.outputs) + for inter_port, outer_port in [("inner_x", "inner_x"), ("inner_t", "inner_t")]: + # Definitions are the same: test two "paths" of accessing the type. + assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME + assert g1.outputs[inter_port].type.compare(g2.outputs[outer_port].type) == NeuralTypeComparisonResult.SAME + # At the same time - those have to be two different port objects! + assert g1.outputs[inter_port] is not g2.outputs[outer_port] + # And different tensors (as those are "internally produced tensors"!) + assert g1.output_tensors[inter_port] is not g2.output_tensors[outer_port] + + @pytest.mark.unit def test_graph_nesting5_topology_copy_one_module_default_inputs(self): """ @@ -286,3 +327,75 @@ def test_graph_nesting6_topology_copy_one_module_manual_inputs(self): assert g1.output_tensors["y_pred"] is y_pred1 assert g2.output_tensors["y_pred"] is y_pred2 assert y_pred1 is not y_pred2 + + + @pytest.mark.unit + def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: manual binding of inputs and outputs, connects to other modules. + """ + ds = RealFunctionDataLayer(n=100, batch_size=1, name="tgn7_ds") + tn = TaylorNet(dim=4, name="tgn7_tn") + loss = MSELoss(name="tgn7_loss") + + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn7_g1") as g1: + # Copy the input type. + g1.inputs["inner_x"] = tn.input_ports["x"] + # Manually bind the input port. + y_pred1 = tn(x=g1.inputs["inner_x"]) + # Manually bind the output port. + g1.outputs["inner_y_pred"] = y_pred1 + + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn7_g2") as g2: + x, y = ds() + y_pred2 = g1(inner_x=x) + lss = loss(predictions=y_pred2, target=y) + + # Check steps. + assert len(g2.steps) == 3 + assert g2.steps[1] == g1.steps[0] + + # Make sure that the modules are the same. + assert len(g2) == 3 + assert g2["tgn7_tn"] is g1["tgn7_tn"] + + # Make sure that inputs are ok. + assert len(g2.inputs) == 0 + + # Check outputs. + assert len(g2.outputs) == 4 + assert g2.output_ports["x"].compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g2.output_ports["y"].compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert g2.output_ports["loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + # The manually bound name! + assert g2.output_ports["inner_y_pred"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + + # Check the output tensors. + assert len(g2.output_tensors) == 4 + assert g2.output_tensors["x"] == x + assert g2.output_tensors["y"] == y + assert g2.output_tensors["loss"] == lss + # The manually bound name! + assert g2.output_tensors["inner_y_pred"] == y_pred2 + + # Check the "internal tensors". + assert y_pred2 is not y_pred1 + assert g2.tensors["tgn7_ds"]["x"] == x + assert g2.tensors["tgn7_ds"]["y"] == y + assert g2.tensors["tgn7_loss"]["loss"] == lss + # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! + assert g2.tensors["tgn7_tn"]["y_pred"] == y_pred2 + + # Update g2: manually bound only one output. + with g2: + g2.outputs["outer_loss"] = lss + + # Make sure that outputs are ok. + assert len(g2.outputs) == 1 + assert g2.output_ports["outer_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert g2.output_tensors["outer_loss"] is lss diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/test_neural_graphs.py index bbc30f3f6d48..982f42afe0b0 100644 --- a/tests/unit/core/test_neural_graphs.py +++ b/tests/unit/core/test_neural_graphs.py @@ -34,7 +34,7 @@ def test_explicit_graph_with_activation(self): Also tests modules access. """ # Create modules. - dl = RealFunctionDataLayer(n=100, batch_size=4, name="dl") + dl = RealFunctionDataLayer(n=10, batch_size=1, name="dl") fx = TaylorNet(dim=4, name="fx") loss = MSELoss(name="loss") @@ -62,7 +62,7 @@ def test_explicit_graph_with_activation(self): def test_explicit_graph_manual_activation(self): """ Tests initialization of an `explicit` graph using `manual` activation. """ # Create modules. - dl = RealFunctionDataLayer(n=100, batch_size=4) + dl = RealFunctionDataLayer(n=10, batch_size=1) fx = TaylorNet(dim=4) # Create the g0 graph. @@ -85,7 +85,7 @@ def test_explicit_graph_manual_activation(self): @pytest.mark.unit def test_default_output_ports(self): """ Tests automatic binding of default output ports. """ - dl = RealFunctionDataLayer(n=10000, batch_size=128) + dl = RealFunctionDataLayer(n=10, batch_size=1) m2 = TaylorNet(dim=4) loss = MSELoss() @@ -95,6 +95,6 @@ def test_default_output_ports(self): # Tests output ports. assert len(g1.output_ports) == 3 - assert g1.output_ports.tensors["x"].compare(x) == NeuralTypeComparisonResult.SAME - assert g1.output_ports.tensors["y"].compare(t) == NeuralTypeComparisonResult.SAME - assert g1.output_ports.tensors["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["x"].compare(x) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y"].compare(t) == NeuralTypeComparisonResult.SAME + assert g1.output_ports["y_pred"].compare(p) == NeuralTypeComparisonResult.SAME From 59b1d757cbd3fe8d36e2de38b71c123d658d9401 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 13:16:21 -0700 Subject: [PATCH 049/106] final tests for graph nesting Signed-off-by: Tomasz Kornuta --- tests/unit/core/test_neural_graph_nesting.py | 103 ++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 78c0855564cb..5fe5a0011294 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -335,7 +335,7 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). Case: manual binding of inputs and outputs, connects to other modules. """ - ds = RealFunctionDataLayer(n=100, batch_size=1, name="tgn7_ds") + ds = RealFunctionDataLayer(n=10, batch_size=1, name="tgn7_ds") tn = TaylorNet(dim=4, name="tgn7_tn") loss = MSELoss(name="tgn7_loss") @@ -399,3 +399,104 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): assert len(g2.outputs) == 1 assert g2.output_ports["outer_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME assert g2.output_tensors["outer_loss"] is lss + + + @pytest.mark.unit + def test_graph_nesting8_topology_copy_two_modules(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: manual binding of inputs and outputs in the inner graph. + """ + ds = RealFunctionDataLayer(n=10, batch_size=1, name="tgn8_ds") + tn = TaylorNet(dim=4, name="tgn8_tn") + loss = MSELoss(name="tgn8_loss") + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn8_g1") as g1: + # Create input port definitions. + g1.inputs["inner_x"] = tn.input_ports["x"] + g1.inputs["inner_target"] = loss.input_ports["target"] + + # Connect modules and bound inputs. + y_pred1 = tn(x=g1.inputs["inner_x"]) + lss1 = loss(predictions=y_pred1, target=g1.inputs["inner_target"]) + + # Manually bind the output ports. + g1.outputs["inner_y_pred"] = y_pred1 + g1.outputs["inner_loss"] = lss1 + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn8_g2") as g2: + x, y = ds() + y_pred2, lss2 = g1(inner_x=x, inner_target=y) + # Manually bind the output ports. + g2.outputs["outer_y_pred"] = y_pred2 + g2.outputs["outer_loss"] = lss2 + + # Check modules and steps. + assert len(g2.steps) == 3 + assert len(g2) == 3 + + # Check the output tensors. + assert len(g2.output_tensors) == 2 + assert g2.output_tensors["outer_y_pred"] == y_pred2 + assert g2.output_tensors["outer_loss"] == lss2 + + # Check the "internal tensors". + assert y_pred2 is not y_pred1 + assert lss2 is not lss1 + assert g2.tensors["tgn8_ds"]["x"] == x + assert g2.tensors["tgn8_ds"]["y"] == y + # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! + assert g2.tensors["tgn8_tn"]["y_pred"] == y_pred2 + # Analogically with "loss". + assert g2.tensors["tgn8_loss"]["loss"] == lss2 + + + @pytest.mark.unit + def test_graph_nesting9_topology_copy_whole_graph(self): + """ + Test whether when nesting of one graph into another will result in copy of the graph topology (tensors). + Case: manual binding of inputs and outputs in the inner graph. Manual binding of outer graph outputs. + """ + ds = RealFunctionDataLayer(n=10, batch_size=1, name="tgn9_ds") + tn = TaylorNet(dim=4, name="tgn9_tn") + loss = MSELoss(name="tgn9_loss") + + # Create the "inner graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn9_g1") as g1: + # Connect modules. + x, y = ds() + y_pred1 = tn(x=x) + lss1 = loss(predictions=y_pred1, target=y) + + # Manually bind the output ports. + g1.outputs["inner_y_pred"] = y_pred1 + g1.outputs["inner_loss"] = lss1 + + # Create the "outer graph". + with NeuralGraph(operation_mode=OperationMode.training, name="tgn9_g2") as g2: + y_pred2, lss2 = g1() + # Manually bind the output ports. + g2.outputs["outer_y_pred"] = y_pred2 + g2.outputs["outer_loss"] = lss2 + + # Check modules and steps. + assert len(g2.steps) == 3 + assert len(g2) == 3 + + # Check the output tensors. + assert len(g2.output_tensors) == 2 + assert g2.output_tensors["outer_y_pred"] == y_pred2 + assert g2.output_tensors["outer_loss"] == lss2 + + # Check the "internal tensors". + assert y_pred2 is not y_pred1 + assert lss2 is not lss1 + assert g2.tensors["tgn9_ds"]["x"].type.compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g2.tensors["tgn9_ds"]["y"].type.compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME + # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! + assert g2.tensors["tgn9_tn"]["y_pred"].type.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + # Analogically with "loss". + assert g2.tensors["tgn9_loss"]["loss"].type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + \ No newline at end of file From fc9d9953e165cea88bcbc754ba9548dbe68b9c46 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 13:16:38 -0700 Subject: [PATCH 050/106] style fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 2 +- tests/unit/core/test_neural_graph_nesting.py | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 1d3cf1ad9527..08056b4b0436 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -111,7 +111,7 @@ def __len__(self): else: # Use default dict. return len(self._default_outputs) - def bind(self, tensors_list, port_names = None): + def bind(self, tensors_list, port_names=None): """ Binds the default outputs. Args: diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/test_neural_graph_nesting.py index 5fe5a0011294..07a501027aaf 100644 --- a/tests/unit/core/test_neural_graph_nesting.py +++ b/tests/unit/core/test_neural_graph_nesting.py @@ -187,7 +187,6 @@ def test_graph_nesting4_topology_copy_one_module_manual_outputs(self): # And different tensors (as those are "internally produced tensors"!) assert g1.output_tensors[inter_port] is not g2.output_tensors[outer_port] - @pytest.mark.unit def test_graph_nesting4_1_topology_copy_one_module_manual_outputs_bound_only_in_inner(self): """ @@ -227,7 +226,6 @@ def test_graph_nesting4_1_topology_copy_one_module_manual_outputs_bound_only_in_ # And different tensors (as those are "internally produced tensors"!) assert g1.output_tensors[inter_port] is not g2.output_tensors[outer_port] - @pytest.mark.unit def test_graph_nesting5_topology_copy_one_module_default_inputs(self): """ @@ -328,7 +326,6 @@ def test_graph_nesting6_topology_copy_one_module_manual_inputs(self): assert g2.output_tensors["y_pred"] is y_pred2 assert y_pred1 is not y_pred2 - @pytest.mark.unit def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): """ @@ -339,7 +336,6 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): tn = TaylorNet(dim=4, name="tgn7_tn") loss = MSELoss(name="tgn7_loss") - # Create the "inner graph". with NeuralGraph(operation_mode=OperationMode.training, name="tgn7_g1") as g1: # Copy the input type. @@ -349,7 +345,6 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): # Manually bind the output port. g1.outputs["inner_y_pred"] = y_pred1 - # Create the "outer graph". with NeuralGraph(operation_mode=OperationMode.training, name="tgn7_g2") as g2: x, y = ds() @@ -400,7 +395,6 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): assert g2.output_ports["outer_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME assert g2.output_tensors["outer_loss"] is lss - @pytest.mark.unit def test_graph_nesting8_topology_copy_two_modules(self): """ @@ -452,7 +446,6 @@ def test_graph_nesting8_topology_copy_two_modules(self): # Analogically with "loss". assert g2.tensors["tgn8_loss"]["loss"] == lss2 - @pytest.mark.unit def test_graph_nesting9_topology_copy_whole_graph(self): """ @@ -496,7 +489,10 @@ def test_graph_nesting9_topology_copy_whole_graph(self): assert g2.tensors["tgn9_ds"]["x"].type.compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME assert g2.tensors["tgn9_ds"]["y"].type.compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! - assert g2.tensors["tgn9_tn"]["y_pred"].type.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert ( + g2.tensors["tgn9_tn"]["y_pred"].type.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + ) # Analogically with "loss". - assert g2.tensors["tgn9_loss"]["loss"].type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - \ No newline at end of file + assert ( + g2.tensors["tgn9_loss"]["loss"].type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + ) From ece11c3ba4ce90b196fee44739df71df24c88546 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 19:02:46 -0700 Subject: [PATCH 051/106] refactored the neural module serialization, made it more modular, so can use its pieces in graph serialization Signed-off-by: Tomasz Kornuta --- .../tutorials/module_custom_configuration.rst | 4 +- .../tutorials/module_custom_configuration.rst | 4 +- examples/start_here/module_configuration.py | 4 +- .../start_here/module_custom_configuration.py | 74 +--- nemo/core/neural_graph/neural_graph.py | 352 ++++++++++-------- nemo/core/neural_modules.py | 178 ++++++--- nemo/utils/object_registry.py | 8 +- 7 files changed, 369 insertions(+), 255 deletions(-) diff --git a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst index 9ba95fec4911..39f7db0540f1 100644 --- a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst @@ -49,8 +49,8 @@ 请注意，基类 :class:`NeuralModule` 提供了一些保护方法供我们使用, \ 其中，最重要的是: - * :meth:`_create_config_header()` 生成合适的 header, 以及 \ - * :meth:`_validate_config_file()` 验证加载的配置文件 (检查 header 内容)。 + * :meth:`__serialize_header()` 生成合适的 header, 以及 \ + * :meth:`__validate_config_file()` 验证加载的配置文件 (检查 header 内容)。 .. note:: diff --git a/docs/sources/source/tutorials/module_custom_configuration.rst b/docs/sources/source/tutorials/module_custom_configuration.rst index da06ad7929d5..10c12bd5a45a 100644 --- a/docs/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/sources/source/tutorials/module_custom_configuration.rst @@ -49,8 +49,8 @@ Analogically, we must overload the :meth:`import_from_config()` method: Please note that the base :class:`NeuralModule` class provides several protected methods that we used, \ with most important being: - * :meth:`_create_config_header()` generating the appropriate header, and \ - * :meth:`_validate_config_file()` validating the loaded configuration file (checking the header content). + * :meth:`__serialize_header()` generating the appropriate header, and \ + * :meth:`__validate_config_file()` validating the loaded configuration file (checking the header content). .. note:: diff --git a/examples/start_here/module_configuration.py b/examples/start_here/module_configuration.py index 91dbca88d830..3bdc9e24840e 100644 --- a/examples/start_here/module_configuration.py +++ b/examples/start_here/module_configuration.py @@ -28,7 +28,7 @@ dl = RealFunctionDataLayer(n=100, f_name="cos", x_lo=-1, x_hi=1, batch_size=128) # Instantiate a simple feed-forward, single layer neural network. -fx = TaylorNet(dim=4) +fx = TaylorNet(dim=4, name="fx") # Instantitate loss. mse_loss = MSELoss() @@ -37,7 +37,7 @@ fx.export_to_config("/tmp/taylor_net.yml") # Create a second instance, using the parameters loaded from the previously created configuration. -fx2 = NeuralModule.import_from_config("/tmp/taylor_net.yml") +fx2 = NeuralModule.import_from_config("/tmp/taylor_net.yml", name="fx2") # Create a graph by connecting the outputs with inputs of modules. x, y = dl() diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index 4b66a84480f8..bf26180b6c4a 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -40,81 +40,47 @@ def __init__(self, dim, status: Status): super().__init__(dim) logging.info("Status: {}".format(status)) - def export_to_config(self, config_file): + def _serialize_configuration(self): """ - A custom method exporting configuration to a YAML file. + A custom method serializing the configuration to a YAML file. - Args: - config_file: path (absolute or relative) and name of the config file (YML) + Returns: + a "serialized" dictionary with module configuration. """ - # Greate an absolute path. - abs_path_file = path.expanduser(config_file) - # Create the dictionary to be exported. - to_export = {} + init_to_export = {} - # Add "header" with module "specification". - to_export["header"] = self._create_config_header() + # "Serialize dim. + init_to_export["dim"] = self._init_params["dim"] - # Add init parameters. - to_export["init_params"] = self._init_params - - # Custom processing of the status. - if to_export["init_params"]["status"] == Status.success: - to_export["init_params"]["status"] = 0 + # Custom "serialization" of the status. + if self._init_params["status"] == Status.success: + init_to_export["status"] = 0 else: - to_export["init_params"]["status"] = 1 - - # All parameters are ok, let's export. - with open(abs_path_file, 'w') as outfile: - yaml.dump(to_export, outfile) - - logging.info( - "Configuration of module {} ({}) exported to {}".format(self._uuid, type(self).__name__, abs_path_file) - ) + init_to_export["status"] = 1 + + # Return serialized params. + return init_to_export @classmethod - def import_from_config(cls, config_file, section_name=None, overwrite_params={}): + def _deserialize_configuration(cls, init_params): """ - A custom method importing the YAML configuration file. - Raises an ImportError exception when config file is invalid or - incompatible (when called from a particular class). + A function that deserializes the module "configuration (i.e. init parameters). Args: - config_file: path (absolute or relative) and name of the config file (YML) - - section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) - - overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) the default - parameters loaded from the configuration file + init_params: List of init parameters loaded from the YAML file. Returns: - Instance of the created NeuralModule object. + A "deserialized" list with init parameters. """ - - # Validate the content of the configuration file (its header). - loaded_config = cls._validate_config_file(config_file, section_name) - - # Get init parameters. - init_params = loaded_config["init_params"] - # Update parameters with additional ones. - init_params.update(overwrite_params) - - # Custom processing of the status. + # Custom "deserialization" of the status. if init_params["status"] == 0: init_params["status"] = Status.success else: init_params["status"] = Status.error - # Create and return the object. - obj = CustomTaylorNet(**init_params) - logging.info( - "Instantiated a new Neural Module of type `{}` using configuration loaded from the `{}` file".format( - "CustomTaylorNet", config_file - ) - ) - return obj + return init_params # Run on CPU. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 1cc4c05beef9..ad37520bf779 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -78,9 +78,10 @@ def __call__(self, **kwargs): This method "nests" one existing neural graph into another one. Also checks if all inputs were provided and properly connects them. - """ - # print(" Neural Graph {} __call__".format(self._name)) + Args: + kwargs: keyword arguments containing dictionary of (input_port_name, port_content). + """ # Test operation modes of the nested graphs. outer_mode = self._app_state.active_graph.operation_mode inner_mode = self.operation_mode @@ -109,150 +110,6 @@ def __call__(self, **kwargs): # Return output tensors. return results - @property - def inputs(self): - """ - Returns graph inputs. - - Returns: - A graph input. - """ - return self._inputs - - @property - def input_ports(self) -> Optional[Dict[str, NeuralType]]: - """ - Returns definitions of graph input ports (dict of Neural Types). - - .. note:: - This method actually returns a dictionary with definitions (like Neural Modules). - In order to get access to actual graph inputs please call the inputs() method. - - Returns: - A graph input ports definitions. - """ - return self._inputs.definitions - - @property - def outputs(self): - """ - Returns graph outputs. - - Returns: - A graph outputs. - """ - return self._outputs - - @property - def output_ports(self) -> Optional[Dict[str, NeuralType]]: - """ - Returns definitions of module output ports (dict of Neural Types). - - .. note:: - This method actually returns a dictionary with definitions (like Neural Modules). - In order to get access to actual graph outpus please call the outputs() method. - - Returns: - A graph output ports definitions. - - """ - return self._outputs.definitions - - @property - def output_tensors(self): - """ - Returns graph output tensors. - - Returns: - A graph output tensors. - """ - return self._outputs.tensors - - @property - def modules(self): - """ Returns modules. """ - return self._modules - - @property - def steps(self): - """ Returns steps. """ - return self._steps - - @property - def operation_mode(self): - """ Returns operation mode. """ - return self._operation_mode - - def __enter__(self): - """ - Activates this graph. - - Returns: - The graph object. - """ - self._app_state.active_graph = self - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - """ - Deactivates the current graph. - """ - self._app_state.active_graph = None - - def activate(self): - """ - Activates this graph. - """ - self._app_state.active_graph = self - - def deactivate(self): - """ - Deactivates the current graph. - """ - self._app_state.active_graph = None - - def __str__(self): - """ Prints a nice summary. """ - # TODO: a nice summary. ;) - desc = "`{}` ({}):\n".format(self.name, len(self._steps)) - for op in self._steps: - desc = desc + " {}\n".format(type(op[0]).__name__) - return desc - - def __getitem__(self, key): - """ Returns module given its name (name of the variable). - - Args: - key: Name of the variable. - """ - if key not in self._modules.keys(): - raise KeyError("Neural Graph doesn't contain a module named {}".format(key)) - return self._modules[key] - - def __len__(self): - return len(self._modules) - - def list_modules(self): - desc = "{} ({}):\n".format(self.name, len(self)) - for key, value in self._modules.items(): - desc += " * `{}` ({})\n".format(key, value) - return desc - - def record_step(self, module): - """ - Records the operation (module plus passed inputs) on a list. - """ - # Check if module with that name already exists - to avoid the potential loops (DAG). - # TODO: Uncomment after we will refactor all the examples, so training/validation graphs won't be added - # to the "default" graph. - # if module.name in self._modules.keys(): - # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) - # Add module to list of modules. - self._modules[module.name] = module - - # Add step - store the module name. - self._steps[len(self._steps)] = module.name - def nest(self, inner_graph, inner_graph_args): """ Method nests (copies) a graph: modules, steps, topology (tensors). @@ -374,6 +231,26 @@ def nest(self, inner_graph, inner_graph_args): # Return the results. return results + def record_step(self, module): + """ + Records the operation (module plus passed inputs) on a list. + + Args: + module: Neural modules added to a given graph. + """ + # Check if module with that name already exists - to avoid potential loops (DAG). + # TODO: Uncomment after we will refactor all the examples, so training/validation graphs won't be added + # to the "default" graph. + # if module.name in self._modules.keys() and self._modules[module.name] is not module: + # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) + + # Add module to list of modules. + self._modules[module.name] = module + + # Add step - store the module name. + self._steps[len(self._steps)] = module.name + + def bind_outputs(self, tensors_list): """ Binds the output tensors. @@ -395,11 +272,192 @@ def bind_outputs(self, tensors_list): if self.default_output_binding: self.outputs.bind(tensors_list) + @property + def inputs(self): + """ + Returns graph inputs. + + Returns: + A graph input. + """ + return self._inputs + + @property + def input_ports(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of graph input ports (dict of Neural Types). + + .. note:: + This method actually returns a dictionary with definitions (like Neural Modules). + In order to get access to actual graph inputs please call the inputs() method. + + Returns: + A graph input ports definitions. + """ + return self._inputs.definitions + + @property + def outputs(self): + """ + Returns graph outputs. + + Returns: + A graph outputs. + """ + return self._outputs + + @property + def output_ports(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports (dict of Neural Types). + + .. note:: + This method actually returns a dictionary with definitions (like Neural Modules). + In order to get access to actual graph outpus please call the outputs() method. + + Returns: + A graph output ports definitions. + + """ + return self._outputs.definitions + + @property + def output_tensors(self): + """ + Returns graph output tensors. + + Returns: + A graph output tensors. + """ + return self._outputs.tensors + + @property + def modules(self): + """ Returns modules. """ + return self._modules + + def __getitem__(self, key): + """ Returns module given its name (name of the variable). + + Args: + key: Name of the variable. + """ + if key not in self._modules.keys(): + raise KeyError("Neural Graph doesn't contain a module named {}".format(key)) + return self._modules[key] + + def __len__(self): + """ Returns number of modules (vertices) in a given graph. """ + return len(self._modules) + + @property + def steps(self): + """ Returns steps. """ + return self._steps + @property def tensors(self): - """ Returns the dictionary of all output tensors, aggregated by modules (key). """ + """ Returns the (double) dictionary of all output tensors, aggregated by modules (key) and (output) port name. """ return self._all_tensors + @property + def operation_mode(self): + """ Returns operation mode. """ + return self._operation_mode + + def __enter__(self): + """ + Activates this graph. + + Returns: + The graph object. + """ + self._app_state.active_graph = self + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + """ + Deactivates the current graph. + """ + self._app_state.active_graph = None + + def activate(self): + """ + Activates this graph. + """ + self._app_state.active_graph = self + + def deactivate(self): + """ + Deactivates the current graph. + """ + self._app_state.active_graph = None + + def ___serialize_header(self): + """ Method serializes the graph header. """ + # Only version and full_spec - for now. + full_spec = str(self.__module__) + "." + str(self.__class__.__qualname__) + header = { + "nemo_core_version": nemo_version, + "full_spec": full_spec + } + return header + + def __serialize_modules(self): + """ Method serializes the modules present in the graph. """ + d = {} + for name, module in self._modules.items(): + d = 1 + + def export_to_config(self, config_file): + """ Exports the neural graph to a file. + + Args: + config_file: Name (and path) of the config file (YML) to be written to. + """ + # Create a dictionary where we will add the whole information. + config = {self.name: {}} + # Get shortcut. + graph = config[self.name] + # Serialize modules. + graph["modules"] = self.__serialize_modules() + + @classmethod + def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite_params={}): + """ + Class method importing the neural graph from the configuration file. + Raises an ImportError exception when config file is invalid. + + Args: + config_file: path (absolute or relative) and name of the config file (YML) + + reuse_existing_modules: If the modules with (name, type, init_params) are already created, import will + connect to them instead of creating new instances. + + overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) the default + parameters loaded from the configuration file + + Returns: + Instance of the created NeuralGraph object. + """ + pass + + + def __str__(self): + """ Prints a nice summary. """ + # TODO: a nice summary. ;) + desc = "`{}` ({}):\n".format(self.name, len(self._steps)) + for op in self._steps: + desc = desc + " {}\n".format(type(op[0]).__name__) + return desc + + def list_modules(self): + """ Lists modules. """ + desc = "{} ({}):\n".format(self.name, len(self._modules)) + for key, value in self._modules.items(): + desc += " * `{}` ({})\n".format(key, value) + return desc + def show_inputs(self): print("bound input ports: ") # for key, value in self._bound_input_ports.items(): diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index c3eea575e34c..2e435ca0913e 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -184,8 +184,52 @@ def __is_of_allowed_type(self, var): # Well, seems that everything is ok. return True - def _create_config_header(self): - """ A protected method that create a header stored later in the configuration file. """ + def export_to_config(self, config_file): + """ + A function that exports module "configuration" (i.e. init parameters) to a YAML file. + Raises a ValueError exception in case then parameters coudn't be exported. + + Args: + config_file: path (absolute or relative) and name of the config file (YML) + """ + # Greate an absolute path. + abs_path_file = path.expanduser(config_file) + + # Serialize the module. + to_export = self.serialize() + + # All parameters are ok, let's export. + with open(abs_path_file, 'w') as outfile: + YAML.dump(to_export, outfile) + + logging.info( + "Configuration of module {} ({}) exported to {}".format(self._uuid, type(self).__name__, abs_path_file) + ) + + def serialize(self): + """ A method serializing the whole Neural module (into a dictionary). + + Returns: + Dictionary containing a "serialized" module. + """ + # Create a dictionary representing the serialized object. + serialized_module = {} + + # Add "header" with module "specification". + serialized_module["header"] = self.__serialize_header() + + # Add init parameters. + serialized_module["init_params"] = self._serialize_configuration() + + # Return dictionary. + return serialized_module + + def __serialize_header(self): + """ A protected method that creates a header stored later in the configuration file. + + Returns: + Dictionary containing a header with module specification. + """ # Get module "full specification". module_full_spec = str(self.__module__) + "." + str(self.__class__.__qualname__) @@ -227,13 +271,16 @@ def _create_config_header(self): } return header - def export_to_config(self, config_file): + def _serialize_configuration(self): """ - A function that exports module "configuration" (i.e. init parameters) to a YAML file. + A function that serializes the module "configuration (i.e. init parameters) to a dictionary. Raises a ValueError exception in case then parameters coudn't be exported. - Args: - config_file: path (absolute or relative) and name of the config file (YML) + ..note: + Thus functions should be overloaded when writing a custom module import/export. + + Returns: + a "serialized" dictionary with module configuration. """ # Check if generic export will work. if not self.__validate_params(self._init_params): @@ -242,30 +289,12 @@ def export_to_config(self, config_file): F"or (lists of/dicts of) primitive types. Please implement your own custom `export_to_config()` and " F"`import_from_config()` methods for your custom Module class." ) + # In this case configuration = init parameters. + return self._init_params - # Greate an absolute path. - abs_path_file = path.expanduser(config_file) - - # Create the dictionary to be exported. - to_export = {} - - # Add "header" with module "specification". - to_export["header"] = self._create_config_header() - - # Add init parameters. - to_export["init_params"] = self._init_params - # print(to_export) - - # All parameters are ok, let's export. - with open(abs_path_file, 'w') as outfile: - YAML.dump(to_export, outfile) - - logging.info( - "Configuration of module {} ({}) exported to {}".format(self._uuid, type(self).__name__, abs_path_file) - ) @classmethod - def _validate_config_file(cls, config_file, section_name=None): + def __validate_config_file(cls, config_file, section_name=None): """ Class method validating whether the config file has a proper content (sections, specification etc.). Raises an ImportError exception when config file is invalid or @@ -319,7 +348,7 @@ def _validate_config_file(cls, config_file, section_name=None): return loaded_config @classmethod - def import_from_config(cls, config_file, section_name=None, overwrite_params={}): + def import_from_config(cls, config_file, section_name=None, name=None, overwrite_params={}): """ Class method importing the configuration file. Raises an ImportError exception when config file is invalid or @@ -330,41 +359,96 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={}) section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) - overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) the default - parameters loaded from the configuration file + overwrite_init_params: Dictionary containing parameters that will be added to or overwrite (!) + the default init parameters loaded from the configuration file (the module "init_params" section). Returns: Instance of the created NeuralModule object. """ + logging.info( + "Loading configuration of a new Neural Module from the `{}` file".format( + config_file + ) + ) # Validate the content of the configuration file (its header). - loaded_config = cls._validate_config_file(config_file, section_name) + loaded_config = cls.__validate_config_file(config_file, section_name) - # Parse the "full specification". - spec_list = loaded_config["header"]["full_spec"].split(".") + # Update parameters with additional ones. + loaded_config["init_params"].update(overwrite_params) - # Get object class from "full specification". - mod_obj = __import__(spec_list[0]) - for spec in spec_list[1:]: - mod_obj = getattr(mod_obj, spec) - # print(mod_obj) + # Override module name in init_params using the logic: + # * section_name if not none overrides init_params.name first (skipped for now, TOTHINK!) + # * name (if None) overrides init_params.name + if name is not None: + loaded_config["init_params"]["name"] = name + + # "Deserialize" the module. + obj = cls.deserialize(loaded_config) + + return obj + + @classmethod + def deserialize(cls, configuration): + """ + Class method instantianting the neural module object based on the configuration (dictionary). + + Args: + configuration: Dictionary containing proper "header" and "init_params" sections. + Returns: + Instance of the created NeuralModule object. + """ + # Deserialize header - get object class. + module_class = cls.__deserialize_header(configuration["header"]) + # Get init parameters. - init_params = loaded_config["init_params"] - # Update parameters with additional ones. - init_params.update(overwrite_params) - # TODO: Add section name as default module name! - # if section_name is not None: - # init_params.update({"name": section_name}) + init_params = cls._deserialize_configuration(configuration["init_params"]) # Create and return the object. - obj = mod_obj(**init_params) + obj = module_class(**init_params) logging.info( - "Instantiated a new Neural Module of type `{}` using configuration loaded from the `{}` file".format( - spec_list[-1], config_file + "Instantiated a new Neural Module named `{}` of type `{}`".format( + obj.name, type(obj).__name__ ) ) return obj + @classmethod + def __deserialize_header(cls, header): + """ Method deserializes the header and extracts the module class. + + Returns: + Class of the module to be created. + """ + # Parse the "full specification". + spec_list = header["full_spec"].split(".") + + # Get module class from the "full specification". + mod_obj = __import__(spec_list[0]) + for spec in spec_list[1:]: + mod_obj = getattr(mod_obj, spec) + # print(mod_obj) + + return mod_obj + + + @classmethod + def _deserialize_configuration(cls, init_params): + """ + A function that deserializes the module "configuration (i.e. init parameters). + + ..note: + Thus functions should be overloaded when writing a custom module import/export. + + Args: + init_params: List of init parameters loaded from the file. + + Returns: + A "deserialized" list with init parameters. + """ + # In this case configuration = init parameters. + return init_params + @deprecated(version=0.11) @staticmethod def create_ports(**kwargs): diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index adfcec7041c3..ef1a5c03a69f 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -109,7 +109,13 @@ def __getitem__(self, key): raise KeyError("A {} with name `{}` don't exists!".format(self._base_type_name, key)) def __eq__(self, other): - """ Checks if two resitrys have similar content. """ + """ Checks if two registers have the same content. """ if not isinstance(other, WeakSet): return False return super().__eq__(other) + + def summary(self): + """ Returns a summary of objects on the list. """ + summary = "Objects:\n" + for obj in self.items(): + summary += " * {} ({})\n".format(obj.name, type(obj).__name) From cd703dec074f90228e0024c1c0d9ee8b51f199f4 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 19:35:25 -0700 Subject: [PATCH 052/106] style fix Signed-off-by: Tomasz Kornuta --- examples/start_here/module_configuration.py | 4 +- .../start_here/module_custom_configuration.py | 18 +++-- nemo/core/neural_graph/neural_graph.py | 78 +++++++++++++++++-- nemo/core/neural_modules.py | 2 +- 4 files changed, 88 insertions(+), 14 deletions(-) diff --git a/examples/start_here/module_configuration.py b/examples/start_here/module_configuration.py index 3bdc9e24840e..42d53ec237ca 100644 --- a/examples/start_here/module_configuration.py +++ b/examples/start_here/module_configuration.py @@ -37,11 +37,11 @@ fx.export_to_config("/tmp/taylor_net.yml") # Create a second instance, using the parameters loaded from the previously created configuration. -fx2 = NeuralModule.import_from_config("/tmp/taylor_net.yml", name="fx2") +fx2 = NeuralModule.import_from_config("/tmp/taylor_net.yml") # Create a graph by connecting the outputs with inputs of modules. x, y = dl() -# Please note that in the graph are using the "second" instance. +# Please note that in the graph we are using the "second" instance. p = fx2(x=x) loss = mse_loss(predictions=p, target=y) diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index bf26180b6c4a..f34c15585ffe 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -51,7 +51,7 @@ def _serialize_configuration(self): # Create the dictionary to be exported. init_to_export = {} - # "Serialize dim. + # "Serialize" dim. init_to_export["dim"] = self._init_params["dim"] # Custom "serialization" of the status. @@ -60,7 +60,7 @@ def _serialize_configuration(self): else: init_to_export["status"] = 1 - # Return serialized params. + # Return serialized parameters. return init_to_export @classmethod @@ -74,13 +74,19 @@ def _deserialize_configuration(cls, init_params): Returns: A "deserialized" list with init parameters. """ + deserialized_params = {} + + # "Deserialize" dim. + deserialized_params["dim"] = init_params["dim"] + # Custom "deserialization" of the status. if init_params["status"] == 0: - init_params["status"] = Status.success + deserialized_params["status"] = Status.success else: - init_params["status"] = Status.error + deserialized_params["status"] = Status.error - return init_params + # Return deserialized parameters. + return deserialized_params # Run on CPU. @@ -104,7 +110,7 @@ def _deserialize_configuration(cls, init_params): # Create a graph by connecting the outputs with inputs of modules. x, y = dl() -# Please note that in the graph are using the "second" instance. +# Please note that in the graph we are using the "second" instance. p = fx2(x=x) loss = mse_loss(predictions=p, target=y) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index ad37520bf779..13ef0e649b4d 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -23,6 +23,7 @@ from collections import OrderedDict, namedtuple from typing import Dict, Optional +from nemo.package_info import __version__ as nemo_version from nemo.core import OperationMode from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs @@ -393,8 +394,40 @@ def deactivate(self): """ self._app_state.active_graph = None - def ___serialize_header(self): - """ Method serializes the graph header. """ + def serialize(self): + """ Method serializes the whole graph. + + Returns: + Dictionary containing description of the whole graph. + """ + # Create a dictionary representing the serialized object. + serialized_graph = {} + + # Add "header" with module "specification". + serialized_graph["header"] = self.__serialize_header() + + # Add modules. + serialized_graph["modules"] = self.__serialize_modules() + + # Add steps. + serialized_graph["steps"] = self.__serialize_steps() + + # Add connectinos. + serialized_graph["connections"] = self.__serialize_connections() + + # Serialize graph (bound) inputs. + # Serialize graph (bound) outputs. + + # Return the dictionary. + return serialized_graph + + + def __serialize_header(self): + """ Private method responsible for serializing the graph header. + + Returns: + Dictionary containing description of the whole graph. + """ # Only version and full_spec - for now. full_spec = str(self.__module__) + "." + str(self.__class__.__qualname__) header = { @@ -404,10 +437,45 @@ def ___serialize_header(self): return header def __serialize_modules(self): - """ Method serializes the modules present in the graph. """ - d = {} + """ Private method responsible for serializing the modules present in the graph. + + Returns: + Dictionary containing description of all graph modules. + """ + serialized_modules = {} for name, module in self._modules.items(): - d = 1 + serialized_modules[name] = module.serialize() + return serialized_modules + + def __serialize_steps(self): + """ Private method responsible for serializing the steps (order of module executions). + + Returns: + Dictionary containing description of the steps. + """ + serialized_steps = {} + for no, module_name in self._steps.items(): + serialized_steps[no] = module_name + return serialized_steps + + def __serialize_connections(self): + """ Private method responsible for serializing the connections in the graph. + + Returns: + List containing "connections" between modules. + """ + serialized_connections = [] + # Iterate through "tensor modules". + for tensors in self._all_tensors.values(): + # Iterate through "tensor output ports". + for tensor in tensors.values(): + # "Transform" tensor to the list of connections. + for c in tensor.connections(): + # Serialize! + source = c.producer.module_name + "." + c.producer.port_name + target = c.consumer.module_name + "." + c.producer.port_name + serialized_connections.append(source + "->" + target) + return serialized_connections def export_to_config(self, config_file): """ Exports the neural graph to a file. diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 2e435ca0913e..f36649be12a9 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -221,7 +221,7 @@ def serialize(self): # Add init parameters. serialized_module["init_params"] = self._serialize_configuration() - # Return dictionary. + # Return the dictionary. return serialized_module def __serialize_header(self): From 465979df888c499bca7d1aae84df6b02b2565ca5 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 22 Apr 2020 19:36:14 -0700 Subject: [PATCH 053/106] graph serialization 80% Signed-off-by: Tomasz Kornuta --- .../start_here/module_custom_configuration.py | 4 ++-- nemo/core/neural_graph/neural_graph.py | 14 ++++---------- nemo/core/neural_modules.py | 16 +++------------- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index f34c15585ffe..793094b12e70 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -59,7 +59,7 @@ def _serialize_configuration(self): init_to_export["status"] = 0 else: init_to_export["status"] = 1 - + # Return serialized parameters. return init_to_export @@ -75,7 +75,7 @@ def _deserialize_configuration(cls, init_params): A "deserialized" list with init parameters. """ deserialized_params = {} - + # "Deserialize" dim. deserialized_params["dim"] = init_params["dim"] diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 13ef0e649b4d..9cd3f99036ab 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -23,12 +23,12 @@ from collections import OrderedDict, namedtuple from typing import Dict, Optional -from nemo.package_info import __version__ as nemo_version from nemo.core import OperationMode from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor +from nemo.package_info import __version__ as nemo_version class NeuralGraph(NeuralInterface): @@ -244,14 +244,13 @@ def record_step(self, module): # to the "default" graph. # if module.name in self._modules.keys() and self._modules[module.name] is not module: # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) - - # Add module to list of modules. + + # Add module to list of modules. self._modules[module.name] = module # Add step - store the module name. self._steps[len(self._steps)] = module.name - def bind_outputs(self, tensors_list): """ Binds the output tensors. @@ -420,7 +419,6 @@ def serialize(self): # Return the dictionary. return serialized_graph - def __serialize_header(self): """ Private method responsible for serializing the graph header. @@ -430,10 +428,7 @@ def __serialize_header(self): """ # Only version and full_spec - for now. full_spec = str(self.__module__) + "." + str(self.__class__.__qualname__) - header = { - "nemo_core_version": nemo_version, - "full_spec": full_spec - } + header = {"nemo_core_version": nemo_version, "full_spec": full_spec} return header def __serialize_modules(self): @@ -510,7 +505,6 @@ def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite """ pass - def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index f36649be12a9..ed5ab60e5c42 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -292,7 +292,6 @@ def _serialize_configuration(self): # In this case configuration = init parameters. return self._init_params - @classmethod def __validate_config_file(cls, config_file, section_name=None): """ @@ -365,11 +364,7 @@ def import_from_config(cls, config_file, section_name=None, name=None, overwrite Returns: Instance of the created NeuralModule object. """ - logging.info( - "Loading configuration of a new Neural Module from the `{}` file".format( - config_file - ) - ) + logging.info("Loading configuration of a new Neural Module from the `{}` file".format(config_file)) # Validate the content of the configuration file (its header). loaded_config = cls.__validate_config_file(config_file, section_name) @@ -400,17 +395,13 @@ def deserialize(cls, configuration): """ # Deserialize header - get object class. module_class = cls.__deserialize_header(configuration["header"]) - + # Get init parameters. init_params = cls._deserialize_configuration(configuration["init_params"]) # Create and return the object. obj = module_class(**init_params) - logging.info( - "Instantiated a new Neural Module named `{}` of type `{}`".format( - obj.name, type(obj).__name__ - ) - ) + logging.info("Instantiated a new Neural Module named `{}` of type `{}`".format(obj.name, type(obj).__name__)) return obj @classmethod @@ -431,7 +422,6 @@ def __deserialize_header(cls, header): return mod_obj - @classmethod def _deserialize_configuration(cls, init_params): """ From ba4646606484693354068c9a04b9f9da80483f5a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 23 Apr 2020 11:55:32 -0700 Subject: [PATCH 054/106] graph serialization operational Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests2_1.py | 17 ++- .../graph_composition_integration_tests3_1.py | 3 + nemo/core/neural_graph/graph_inputs.py | 17 +++ nemo/core/neural_graph/graph_outputs.py | 18 +++ nemo/core/neural_graph/neural_graph.py | 42 ++++--- nemo/core/neural_modules.py | 109 +++++++++--------- 6 files changed, 133 insertions(+), 73 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index cb238910ca5f..5415c8f31e07 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -39,18 +39,16 @@ x, t = dl() prediction1 = m1(x=x) prediction2 = m2(x=x) - # Manually bind the selected output ports. - g1.output_ports["ix"] = x - g1.output_ports["te"] = t - g1.output_ports["prediction"] = prediction2 + # Manually bind the selected outputs. + g1.outputs["ix"] = x + g1.outputs["te"] = t + g1.outputs["prediction"] = prediction2 with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g2: x1, t1, p1 = g1() lss = loss(predictions=p1, target=t1) -pdb.set_trace() - # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), @@ -58,3 +56,10 @@ # Invoke "train" action. nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") + +# Serialize graph +serialized_g1 = g1.serialize() +print() + +# Deserialize graph. +g3 = NeuralGraph.deserialize(serialized_g1) diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index aa687ed11cb0..0d982026356e 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -56,3 +56,6 @@ # Invoke "train" action. nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") + +# Serialize graph +print(g2.serialize()) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 74d6271be8f4..12df57fabfa2 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -126,3 +126,20 @@ def has_binding(self, module_name, port_name): if module == module_name and port == port_name: return key return None + + + def serialize(self): + """ Method responsible for serialization of the graph inputs. + + Returns: + List containing mappings (input -> module.input_port). + """ + serialized_inputs = [] + # Iterate through "bindings". + for key, binding in self._inputs.items(): + for (module, port) in binding.consumers_ports: + # Serialize: input -> module.port. + target = module + "." + port + serialized_inputs.append(key + "->" + target) + # Return the result. + return serialized_inputs diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 08056b4b0436..9116dcacba57 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -162,3 +162,21 @@ def tensors(self): output_tensors[k] = tensor return output_tensors + + def serialize(self): + """ Method responsible for serialization of the graph outputs. + + Returns: + List containing mappings (module.output_port -> output). + """ + serialized_outputs = {"default": [], "manual":[]} + + # Serialize both dictionaries - for now. + for d, name in [(self._default_outputs, "default"), (self._manual_outputs, "manual")]: + # Iterate through "bindings". + for key, binding in d.items(): + # Serialize: module.port -> output. + source = binding.producer_port.module_name + "." + binding.producer_port.port_name + serialized_outputs[name].append(source + "->" + key) + # Return the result. + return serialized_outputs diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 9cd3f99036ab..e9da1346ef1b 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -393,6 +393,19 @@ def deactivate(self): """ self._app_state.active_graph = None + def export_to_config(self, config_file): + """ Exports the neural graph to a file. + + Args: + config_file: Name (and path) of the config file (YML) to be written to. + """ + # Create a dictionary where we will add the whole information. + config = {self.name: {}} + # Get shortcut. + graph = config[self.name] + # Serialize modules. + graph["modules"] = self.__serialize_modules() + def serialize(self): """ Method serializes the whole graph. @@ -415,7 +428,10 @@ def serialize(self): serialized_graph["connections"] = self.__serialize_connections() # Serialize graph (bound) inputs. + serialized_graph["inputs"] = self._inputs.serialize() + # Serialize graph (bound) outputs. + serialized_graph["outputs"] = self._outputs.serialize() # Return the dictionary. return serialized_graph @@ -426,9 +442,17 @@ def __serialize_header(self): Returns: Dictionary containing description of the whole graph. """ - # Only version and full_spec - for now. + # Generate full_spec of the class. full_spec = str(self.__module__) + "." + str(self.__class__.__qualname__) header = {"nemo_core_version": nemo_version, "full_spec": full_spec} + # Add operation mode. + if self._operation_mode == OperationMode.training: + header["operation_mode"] = "training" + if self._operation_mode == OperationMode.inference: + header["operation_mode"] = "inference" + else: + header["operation_mode"] = "both" + # Return header. return header def __serialize_modules(self): @@ -472,18 +496,10 @@ def __serialize_connections(self): serialized_connections.append(source + "->" + target) return serialized_connections - def export_to_config(self, config_file): - """ Exports the neural graph to a file. - - Args: - config_file: Name (and path) of the config file (YML) to be written to. - """ - # Create a dictionary where we will add the whole information. - config = {self.name: {}} - # Get shortcut. - graph = config[self.name] - # Serialize modules. - graph["modules"] = self.__serialize_modules() + @classmethod + def deserialize(cls, configuration, reuse_existing_modules=False, overwrite_params={}): + pass + @classmethod def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite_params={}): diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index ed5ab60e5c42..e15422865832 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -292,60 +292,6 @@ def _serialize_configuration(self): # In this case configuration = init parameters. return self._init_params - @classmethod - def __validate_config_file(cls, config_file, section_name=None): - """ - Class method validating whether the config file has a proper content (sections, specification etc.). - Raises an ImportError exception when config file is invalid or - incompatible (when called from a particular class). - - Args: - config_file: path (absolute or relative) and name of the config file (YML) - - section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) - - Returns: - A loaded configuration file (dictionary). - """ - # Greate an absolute path. - abs_path_file = path.expanduser(config_file) - - # Open the config file. - with open(abs_path_file, 'r') as stream: - loaded_config = YAML.load(stream) - - # Check section. - if section_name is not None: - if section_name not in loaded_config: - raise ImportError( - "The loaded config `{}` doesn't contain the indicated `{}` section".format( - config_file, section_name - ) - ) - # Section exists - use only it for configuration. - loaded_config = loaded_config[section_name] - - # Make sure that the config is valid. - if "header" not in loaded_config: - raise ImportError("The loaded config `{}` doesn't contain the `header` section".format(config_file)) - - if "init_params" not in loaded_config: - raise ImportError("The loaded config `{}` doesn't contain the `init_params` section".format(config_file)) - - # Parse the "full specification". - spec_list = loaded_config["header"]["full_spec"].split(".") - - # Check if config contains data of a compatible class. - if cls.__name__ != "NeuralModule" and spec_list[-1] != cls.__name__: - txt = "The loaded file `{}` contains configuration of ".format(config_file) - txt = txt + "`{}` thus cannot be used for instantiation of an object of type `{}`".format( - spec_list[-1], cls.__name__ - ) - raise ImportError(txt) - - # Success - return configuration. - return loaded_config - @classmethod def import_from_config(cls, config_file, section_name=None, name=None, overwrite_params={}): """ @@ -402,6 +348,7 @@ def deserialize(cls, configuration): # Create and return the object. obj = module_class(**init_params) logging.info("Instantiated a new Neural Module named `{}` of type `{}`".format(obj.name, type(obj).__name__)) + return obj @classmethod @@ -439,6 +386,60 @@ def _deserialize_configuration(cls, init_params): # In this case configuration = init parameters. return init_params + @classmethod + def __validate_config_file(cls, config_file, section_name=None): + """ + Class method validating whether the config file has a proper content (sections, specification etc.). + Raises an ImportError exception when config file is invalid or + incompatible (when called from a particular class). + + Args: + config_file: path (absolute or relative) and name of the config file (YML) + + section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) + + Returns: + A loaded configuration file (dictionary). + """ + # Greate an absolute path. + abs_path_file = path.expanduser(config_file) + + # Open the config file. + with open(abs_path_file, 'r') as stream: + loaded_config = YAML.load(stream) + + # Check section. + if section_name is not None: + if section_name not in loaded_config: + raise ImportError( + "The loaded config `{}` doesn't contain the indicated `{}` section".format( + config_file, section_name + ) + ) + # Section exists - use only it for configuration. + loaded_config = loaded_config[section_name] + + # Make sure that the config is valid. + if "header" not in loaded_config: + raise ImportError("The loaded config `{}` doesn't contain the `header` section".format(config_file)) + + if "init_params" not in loaded_config: + raise ImportError("The loaded config `{}` doesn't contain the `init_params` section".format(config_file)) + + # Parse the "full specification". + spec_list = loaded_config["header"]["full_spec"].split(".") + + # Check if config contains data of a compatible class. + if cls.__name__ != "NeuralModule" and spec_list[-1] != cls.__name__: + txt = "The loaded file `{}` contains configuration of ".format(config_file) + txt = txt + "`{}` thus cannot be used for instantiation of an object of type `{}`".format( + spec_list[-1], cls.__name__ + ) + raise ImportError(txt) + + # Success - return configuration. + return loaded_config + @deprecated(version=0.11) @staticmethod def create_ports(**kwargs): From fa1e4b3a06487d1f93e667ffbec754a759e79638 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 23 Apr 2020 20:04:51 -0700 Subject: [PATCH 055/106] work on deserialization, working up to the execution of graph Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests2_1.py | 34 ++- .../graph_composition_integration_tests3_1.py | 25 +- nemo/core/neural_graph/graph_inputs.py | 1 - nemo/core/neural_graph/graph_outputs.py | 4 +- nemo/core/neural_graph/neural_graph.py | 221 +++++++++++++++++- nemo/core/neural_modules.py | 37 +-- nemo/utils/object_registry.py | 26 ++- 7 files changed, 289 insertions(+), 59 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index 5415c8f31e07..a81982f0a911 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -44,22 +44,36 @@ g1.outputs["te"] = t g1.outputs["prediction"] = prediction2 +# Serialize graph +serialized_g1 = g1.serialize() +print(serialized_g1) + +# Delete everything! +del g1 +del dl +del m1 +del m2 +del loss + +# Deserialize graph. +g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g1_copy") + with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g2: - x1, t1, p1 = g1() + x1, t1, p1 = g1_copy() lss = loss(predictions=p1, target=t1) + # Manual output. + g2.outputs["loss"] = lss # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), + tensors=[g2.output_tensors["loss"]], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), ) # Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") - -# Serialize graph -serialized_g1 = g1.serialize() -print() - -# Deserialize graph. -g3 = NeuralGraph.deserialize(serialized_g1) +nf.train( + [g2.output_tensors["loss"]], + callbacks=[callback], + optimization_params={"num_epochs": 2, "lr": 0.0003}, + optimizer="sgd", +) diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index 0d982026356e..2454de0dcfe6 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -23,9 +23,9 @@ nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32) -fx = TaylorNet(dim=4) -loss = MSELoss() +dl = RealFunctionDataLayer(n=100, batch_size=32, name="real_function_dl") +fx = TaylorNet(dim=4, name="taylor_net") +loss = MSELoss(name="mse_loss") logging.info( "This example shows how one can nest one graph into another - with binding of the input ports." @@ -34,19 +34,20 @@ F" and then set to `x` returned by `dl` in the graph `g3`." ) -with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: +with NeuralGraph(operation_mode=OperationMode.training, name="model") as model: # Manually bind input port: "input" -> "x" - g2.inputs["input"] = fx.input_ports["x"] + model.inputs["input"] = fx.input_ports["x"] # Add module to graph and bind it input port 'x'. - y = fx(x=g2.inputs["input"]) - # lss = loss(predictions=y, target=g2.input_ports["input"]) + y = fx(x=model.inputs["input"]) + # Manual output bind. + model.outputs["output"] = y # Build the training graph. -with NeuralGraph(operation_mode=OperationMode.training, name="g3") as g3: +with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: # Add modules to graph. x, t = dl() - # Incorporate modules from the existing graph. - p = g2(input=x) + # Incorporate modules from the existing "model" graph. + p = model(input=x) lss = loss(predictions=p, target=t) # SimpleLossLoggerCallback will print loss values to console. @@ -58,4 +59,6 @@ nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") # Serialize graph -print(g2.serialize()) +print(model.serialize()) + +print(training.serialize()) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 12df57fabfa2..6060e01c3177 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -127,7 +127,6 @@ def has_binding(self, module_name, port_name): return key return None - def serialize(self): """ Method responsible for serialization of the graph inputs. diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 9116dcacba57..fbf3fa96112d 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -169,10 +169,10 @@ def serialize(self): Returns: List containing mappings (module.output_port -> output). """ - serialized_outputs = {"default": [], "manual":[]} + serialized_outputs = {"default": [], "manual": []} # Serialize both dictionaries - for now. - for d, name in [(self._default_outputs, "default"), (self._manual_outputs, "manual")]: + for d, name in [(self._default_outputs, "default"), (self._manual_outputs, "manual")]: # Iterate through "bindings". for key, binding in d.items(): # Serialize: module.port -> output. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index e9da1346ef1b..e3360da68fc0 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -24,12 +24,14 @@ from typing import Dict, Optional from nemo.core import OperationMode -from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs +from nemo.core.neural_modules import NeuralModule +from nemo.core.neural_graph.graph_inputs import GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version - +from nemo.utils import logging +from nemo.utils.module_port import ModulePort, Connection class NeuralGraph(NeuralInterface): """ @@ -448,7 +450,7 @@ def __serialize_header(self): # Add operation mode. if self._operation_mode == OperationMode.training: header["operation_mode"] = "training" - if self._operation_mode == OperationMode.inference: + elif self._operation_mode == OperationMode.inference: header["operation_mode"] = "inference" else: header["operation_mode"] = "both" @@ -497,12 +499,7 @@ def __serialize_connections(self): return serialized_connections @classmethod - def deserialize(cls, configuration, reuse_existing_modules=False, overwrite_params={}): - pass - - - @classmethod - def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite_params={}): + def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite_params={}, name=None): """ Class method importing the neural graph from the configuration file. Raises an ImportError exception when config file is invalid. @@ -516,11 +513,217 @@ def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) the default parameters loaded from the configuration file + name: Name of the new graph (optional, DEFAULT: NONE) + + Returns: + Instance of the created NeuralGraph object. + """ + logging.info("Loading configuration of a new Neural Graph from the `{}` file".format(config_file)) + + # TODO: validate the content of the configuration file (its header). + loaded_config = [] # cls.__validate_config_file(config_file, section_name) + # TODO: overwrite params. + + # "Deserialize" the graph. + new_graph = cls.deserialize(loaded_config, reuse_existing_modules, name) + + return new_graph + + @classmethod + def deserialize(cls, configuration, reuse_existing_modules=False, name=None): + """ + Class method creating a graph instance by deserializing the provided configuratino. + + Args: + configuration: Dictionary containing serialized graph. + + reuse_existing_modules: If the modules with (name, type, init_params) are already created, import will + connect to them instead of creating new instances. + Returns: Instance of the created NeuralGraph object. """ + # Deserialize header and get object class. + operation_mode = cls.__deserialize_header(configuration["header"]) + + # Create the graph instance. + new_graph = NeuralGraph(operation_mode=operation_mode, name=name) + logging.info( + "Instantiated a new Neural Graph named `{}` with mode `{}`".format( + new_graph.name, new_graph.operation_mode + ) + ) + # Deserialize modules. + modules = new_graph.__deserialize_modules(configuration["modules"], reuse_existing_modules) + + # Deserialize steps. + steps = new_graph.__deserialize_steps(configuration["steps"]) + + # Deserialize the connections between modules. + connnections = new_graph.__deserialize_connections(configuration["connections"]) + + # TODO Deserialize input bindings. + # new_graph.__deserialize_inputs(configuration["inputs"]) + + # Now we have to execute the graph, following the steps and connections. + + + # TODO: deserialize output bindings. + + # Return the graph instance. + return new_graph + + @classmethod + def __deserialize_header(cls, serialized_header): + """ Private class method deserializing the header and extracts the general information. + + Args: + serialized_header: Dictionary containing graph header. + + Returns: + Operation mode. + """ + # Parse the "full specification" - do not need that now. + # spec_list = serialized_header["full_spec"].split(".") + + # Get operation mode. + if serialized_header["operation_mode"] == "training": + operation_mode = OperationMode.training + elif header["operation_mode"] == "inference": + operation_mode = OperationMode.inference + else: + operation_mode = OperationMode.both + + # Return the mode. + return operation_mode + + def __deserialize_modules(self, serialized_modules, reuse_existing_modules): + """ Private method deserializing the modules present in the graph. + + Args: + serialized_modules: Dictionary containing graph modules. + + Returns: + Dictionary of modules. + """ + modules = {} + for name, module_params in serialized_modules.items(): + # Check if module already exists. + if self._app_state.modules.has(name): + # Check if we can reuse the existing modules. + if reuse_existing_modules: + modules[name] = self._app_state.modules[name] + else: + raise KeyError("A module with name `{}` already exists!".format(name)) + else: + # Ok, create a new module. + modules[name] = NeuralModule.deserialize(module_params) + # Ok, done. + return modules + + def __deserialize_steps(self, serialized_steps): + """ Private method deserializing the steps (order of module executions). + + Args: + serialized_steps: Dictionary containing serialized steps. + + Returns: + Odered dict with steps. + """ + steps = OrderedDict() + for i in range(len(serialized_steps)): + steps[i] = serialized_steps[i] + # Ok, done. + return steps + + def __deserialize_connections(self, serialized_connections): + """ Private method deserializing the connections in the graph. + + Args: + serialized_steps: Dictionary containing serialized connections. + + Returns: + List of connections, in a format enabling graph traversing. + """ + connections = [] + # Deserialize connections one by one. + for c in serialized_connections: + # Deserialize! + [producer,consumer] = c.split("->") + [producer_name, producer_port_name] = producer.split(".") + [consumer_name, consumer_port_name] = consumer.split(".") + producer_mp = ModulePort(producer_name, producer_port_name) + consumer_mp = ModulePort(consumer_name, consumer_port_name) + # Add connection. + connections.append(Connection(producer_mp, consumer_mp)) + # Ok, done. + return connections + + def __execute_and_create_tensors(self, steps, modules, connections, inputs): + """ Method creates (internal) tensors of the graph by executing it following the order. """ + + # Activate this graph, so all the tensors will be added to this ! + self.activate() + + # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce + # them only for ports that are already indicated as the "bound" ones in the inner graph. + self.default_output_binding = False + + # Now "copy" graph execution order and topology by actually executing each step of the nested graph. + for _, module_name in steps.items(): + # Both module and step will be added by the modules' call(). + + # Get the module. + module = modules[module_name] + + # Produce list of arguments that will be passed to a given module. + module_args = {} + # Do it by: + # - harvesing input port names of a given module, + # - checking if the input was not bound (in the inner graph), + # - checking if we have already tensors leading to that input (in outer graph). + for input_port_name in module.input_ports.keys(): + # Check if this port was bound in the inner graph. + key = inputs.has_binding(module_name, input_port_name) + + # If so, then we must pass the binding! + if key is not None: + module_args[input_port_name] = new_graph.inputs[key] + continue + + # Else: find a tensor that should be passed to the given module's input. + # Search for producer/port that we should use. + for connection in connections: + if ( + connection.consumer.module_name == module_name + and connection.consumer.port_name == input_port_name + ): + # Got the connection! + producer_name = connection.producer.module_name + producer_port_name = connection.producer.port_name + break + # Now, the tensor is already produced in outer (i.e. this) graph! + module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + + # import pdb;pdb.set_trace() + # Ok, now we have all keyword arguments. We can call() the module. + # This will collect all the produced output tensors and add them to this graph. + module(**module_args) + + + + def __deserialize_inputs(self, serialized_inputs): + """ Private method deserializing the bound graph inputs. + + Args: + serialized_steps: Dictionary containing serialized inputs. + """ + # TODO! pass + + + def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index e15422865832..aa70991f2248 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -28,8 +28,7 @@ from ruamel.yaml import YAML -from nemo.core import NeuralGraph, NeuralModuleFactory, OperationMode -from nemo.core.neural_graph.graph_inputs import GraphInput +from nemo.core import NeuralModuleFactory, OperationMode from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version @@ -345,32 +344,40 @@ def deserialize(cls, configuration): # Get init parameters. init_params = cls._deserialize_configuration(configuration["init_params"]) - # Create and return the object. - obj = module_class(**init_params) - logging.info("Instantiated a new Neural Module named `{}` of type `{}`".format(obj.name, type(obj).__name__)) - - return obj + # Create the module instance. + new_module = module_class(**init_params) + logging.info( + "Instantiated a new Neural Module named `{}` of type `{}`".format( + new_module.name, type(new_module).__name__ + ) + ) + + # Return the module instance. + return new_module @classmethod - def __deserialize_header(cls, header): + def __deserialize_header(cls, serialized_header): """ Method deserializes the header and extracts the module class. + Args: + serialized_header: Dictionary containing module header. + Returns: Class of the module to be created. """ # Parse the "full specification". - spec_list = header["full_spec"].split(".") + spec_list = serialized_header["full_spec"].split(".") # Get module class from the "full specification". mod_obj = __import__(spec_list[0]) for spec in spec_list[1:]: mod_obj = getattr(mod_obj, spec) - # print(mod_obj) + # Return "class". return mod_obj @classmethod - def _deserialize_configuration(cls, init_params): + def _deserialize_configuration(cls, serialized_init_params): """ A function that deserializes the module "configuration (i.e. init parameters). @@ -378,13 +385,13 @@ def _deserialize_configuration(cls, init_params): Thus functions should be overloaded when writing a custom module import/export. Args: - init_params: List of init parameters loaded from the file. + serialized_init_params: List of init parameters loaded from the file. Returns: A "deserialized" list with init parameters. """ # In this case configuration = init parameters. - return init_params + return serialized_init_params @classmethod def __validate_config_file(cls, config_file, section_name=None): @@ -545,7 +552,7 @@ def __call__(self, **kwargs): # * NmTensor -> check definition, add self as a "consumer" of a tensor (produced by other module). # Check what was actually passed. - if type(port_content) is NeuralGraph: + if type(port_content).__name__ == "NeuralGraph": # Make sure that port_content is the currently active graph! if port_content is not self._app_state.active_graph: raise ConnectionError("Ports can be bound only by passing the active graph object!") @@ -564,7 +571,7 @@ def __call__(self, **kwargs): # Please note that there are no "consumers" here - this is a "pure binding". - elif type(port_content) is GraphInput: + elif type(port_content).__name__ == "GraphInput": # Check if GraphInput belongs to the active graph ! own_port = False diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index ef1a5c03a69f..c35ad878c076 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -54,9 +54,8 @@ def register(self, new_obj, name): unique_name = self.__generate_unique_name() else: # Check if name is unique. - for obj in self: - if obj.name == name: - raise NameError("A {} with name `{}` already exists!".format(self._base_type_name, name)) + if self.has(name): + raise NameError("A {} with name `{}` already exists!".format(self._base_type_name, name)) # Ok, it is unique. unique_name = name @@ -66,6 +65,15 @@ def register(self, new_obj, name): # Return the name. return unique_name + def has(self, name): + """ Check if registry stores object with a given name. """ + for obj in self: + if obj.name == name: + return True + # Else: + return False + + def __generate_unique_name(self): """ Generates a new unique name by adding postfix (number) to base name. @@ -75,17 +83,13 @@ def __generate_unique_name(self): """ # Iterate through numbers. postfix = 0 - name_unique = False - while not name_unique: + while True: # Generate name. new_name = self._base_type_name + str(postfix) - name_unique = True # Check uniqueneess. - for obj in self: - if obj.name == new_name: - # Sadly name is not unique. - name_unique = False - break + if not self.has(new_name): + # Ok, got a unique name! + break # Increment index. postfix += 1 return new_name From b81024376d52472e6544ec60c2501cbc04bf2a93 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 23 Apr 2020 20:05:22 -0700 Subject: [PATCH 056/106] format fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/neural_graph.py | 13 ++++--------- nemo/utils/object_registry.py | 1 - 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index e3360da68fc0..0fdaa0685456 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -24,14 +24,15 @@ from typing import Dict, Optional from nemo.core import OperationMode -from nemo.core.neural_modules import NeuralModule from nemo.core.neural_graph.graph_inputs import GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface +from nemo.core.neural_modules import NeuralModule from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.module_port import ModulePort, Connection +from nemo.utils.module_port import Connection, ModulePort + class NeuralGraph(NeuralInterface): """ @@ -567,7 +568,6 @@ def deserialize(cls, configuration, reuse_existing_modules=False, name=None): # Now we have to execute the graph, following the steps and connections. - # TODO: deserialize output bindings. # Return the graph instance. @@ -649,7 +649,7 @@ def __deserialize_connections(self, serialized_connections): # Deserialize connections one by one. for c in serialized_connections: # Deserialize! - [producer,consumer] = c.split("->") + [producer, consumer] = c.split("->") [producer_name, producer_port_name] = producer.split(".") [consumer_name, consumer_port_name] = consumer.split(".") producer_mp = ModulePort(producer_name, producer_port_name) @@ -710,8 +710,6 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # This will collect all the produced output tensors and add them to this graph. module(**module_args) - - def __deserialize_inputs(self, serialized_inputs): """ Private method deserializing the bound graph inputs. @@ -721,9 +719,6 @@ def __deserialize_inputs(self, serialized_inputs): # TODO! pass - - - def __str__(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index c35ad878c076..229bb70088d1 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -72,7 +72,6 @@ def has(self, name): return True # Else: return False - def __generate_unique_name(self): """ From 9f46e20a0ffbd20e9c2aee5e1d6db4f6b487eb8c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 12:58:35 -0700 Subject: [PATCH 057/106] serialization and deserialization of JASPER (processor, encoder and decoder) working Signed-off-by: Tomasz Kornuta --- ...osition_integration_tests0_jasper_named.py | 36 +++++++- .../graph_composition_integration_tests2_1.py | 16 ++-- .../graph_composition_integration_tests3_1.py | 24 ++++-- nemo/core/neural_graph/graph_inputs.py | 39 ++++++++- nemo/core/neural_graph/graph_outputs.py | 58 +++++++++++-- nemo/core/neural_graph/neural_graph.py | 84 ++++++++++++------- nemo/core/neural_modules.py | 6 +- 7 files changed, 203 insertions(+), 60 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py index 946454573c1d..c5a98a9d3aae 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -25,7 +25,7 @@ import nemo import nemo.collections.asr as nemo_asr from nemo.collections.asr.helpers import monitor_asr_train_progress -from nemo.core import NeuralGraph +from nemo.core import NeuralGraph, OperationMode from nemo.utils import logging from nemo.utils.app_state import AppState @@ -67,16 +67,46 @@ greedy_decoder = nemo_asr.GreedyCTCDecoder() # Create the Jasper composite module. -with NeuralGraph() as Jasper: +with NeuralGraph(operation_mode=OperationMode.both) as Jasper: i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) i_log_probs = jasper_decoder(encoder_output=i_encoded) # All output ports are bind (for now!) +# Serialize graph +serialized_jasper = Jasper.serialize() +print("Serialized:\n", serialized_jasper) + +# 'connections': +# * ['module1.processed_signal->module2.processed_signal', +# * 'module1.processed_length->module2.processed_length', +# * 'module2.outputs->module3.outputs'], +# 'inputs': +# * ['input_signal->module1.input_signal', +# * 'length->module1.length'], +# 'outputs': +# * {'outputs': ['module1.processed_signal->processed_signal', +# * 'module1.processed_length->processed_length', +# * 'module2.outputs->outputs', +# * 'module2.encoded_lengths->encoded_lengths', +# * 'module3.output->output'] + +# Delete everything! +del Jasper +del jasper_encoder +del jasper_decoder +del data_preprocessor + +# Deserialize graph. +jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") +serialized_jasper_copy = jasper_copy.serialize() +print("Deserialized:\n", serialized_jasper_copy) +assert serialized_jasper == serialized_jasper_copy + with NeuralGraph(name="training") as training: # Create the "implicit" training graph. o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() # Use Jasper module as any other neural module. - o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = Jasper( + o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = jasper_copy( input_signal=o_audio_signal, length=o_audio_signal_len ) o_predictions = greedy_decoder(log_probs=o_log_probs) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index a81982f0a911..860ee1cb8179 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -46,17 +46,19 @@ # Serialize graph serialized_g1 = g1.serialize() -print(serialized_g1) +print("Serialized:\n", serialized_g1) # Delete everything! -del g1 -del dl -del m1 -del m2 -del loss +#del g1 +#del dl +#del m1 +#del m2 # Deserialize graph. -g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g1_copy") +g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=True, name="g1_copy") +serialized_g1_copy = g1_copy.serialize() +print("Deserialized:\n", serialized_g1_copy) + with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g2: x1, t1, p1 = g1_copy() diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index 2454de0dcfe6..ac074651543c 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -34,7 +34,8 @@ F" and then set to `x` returned by `dl` in the graph `g3`." ) -with NeuralGraph(operation_mode=OperationMode.training, name="model") as model: +# Create "model". +with NeuralGraph(operation_mode=OperationMode.both, name="model") as model: # Manually bind input port: "input" -> "x" model.inputs["input"] = fx.input_ports["x"] # Add module to graph and bind it input port 'x'. @@ -42,12 +43,26 @@ # Manual output bind. model.outputs["output"] = y +# Serialize graph +serialized_model = model.serialize() +print("Serialized:\n", serialized_model) + +# Delete everything! +#del model +#del fx + +# Deserialize graph. +model_copy = NeuralGraph.deserialize(serialized_model, reuse_existing_modules=True, name="model_copy") + +serialized_model_copy = model_copy.serialize() +print("Deserialized:\n", serialized_model_copy) + # Build the training graph. with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: # Add modules to graph. x, t = dl() # Incorporate modules from the existing "model" graph. - p = model(input=x) + p = model_copy(input=x) lss = loss(predictions=p, target=t) # SimpleLossLoggerCallback will print loss values to console. @@ -59,6 +74,5 @@ nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") # Serialize graph -print(model.serialize()) - -print(training.serialize()) +#print(model.serialize()) +#print(training.serialize()) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 6060e01c3177..69ce4611fb86 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -46,8 +46,12 @@ def bind(self, module_ports): """ Binds the (modules-ports) to this "graph input". Args: - module_ports: List of ModulePort tuples to be added. + module_ports: A single ModulePort OR a list of ModulePort tuples to be added. """ + # Handle both single port and lists of ports to be bound. + if type(module_ports) is not list: + module_ports = [module_ports] + # Interate through "consumers" on the list and add them to bound input. for module_port in module_ports: self._consumers.append(module_port) @@ -87,6 +91,7 @@ def __setitem__(self, key, value): val_type = value elif isinstance(value, GraphInput): val_type = value.type + else: raise TypeError("Port `{}` definition must be must be a NeuralType or GraphInput type".format(key)) # Ok, add definition to list of mapped (module, port)s. # Note: for now, there are no mapped modules, so copy only (neural) type. @@ -142,3 +147,35 @@ def serialize(self): serialized_inputs.append(key + "->" + target) # Return the result. return serialized_inputs + + @classmethod + def deserialize(cls, serialized_inputs, modules, definitions_only=False): + """ + Class method responsible for deserialization of graph inputs. + + Args: + serialized_inputs: A list of serialized inputs in the form of ("input->module.input_port") + modules: List of modules required for neural type copying/checking. + definitions_only: deserializes/checks only the definitions, without binding the consumers. + + Returns: + Dictionary with deserialized inputs. + """ + inputs = GraphInputs() + # Iterate through serialized inputs one by one. + for i in serialized_inputs: + # Deserialize! + [key, consumer] = i.split("->") + [consumer_name, consumer_port_name] = consumer.split(".") + # Add the input. + if key not in inputs.keys(): + # Get neural type from module input port definition. + n_type = modules[consumer_name].input_ports[consumer_port_name] + # Create a new input. + inputs[key] = n_type + # Optionally, also bind the "consumers". + if not definitions_only: + # Bound input. + inputs[key].bind(ModulePort(consumer_name, consumer_port_name)) + # Done. + return inputs \ No newline at end of file diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index fbf3fa96112d..cccc69bd6440 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -19,6 +19,7 @@ from collections.abc import MutableMapping from nemo.utils import logging +from nemo.utils.module_port import ModulePort class GraphOutput(object): @@ -169,14 +170,53 @@ def serialize(self): Returns: List containing mappings (module.output_port -> output). """ - serialized_outputs = {"default": [], "manual": []} - - # Serialize both dictionaries - for now. - for d, name in [(self._default_outputs, "default"), (self._manual_outputs, "manual")]: - # Iterate through "bindings". - for key, binding in d.items(): - # Serialize: module.port -> output. - source = binding.producer_port.module_name + "." + binding.producer_port.port_name - serialized_outputs[name].append(source + "->" + key) + serialized_outputs = {"outputs": []} + + # Get the right output dictionary. + if len(self._manual_outputs) > 0: + serialized_outputs["type"] = "manual" + d = self._manual_outputs + else: + serialized_outputs["type"] = "default" + d = self._default_outputs + + # Iterate through "bindings". + for key, binding in d.items(): + # Serialize: module.port -> output. + source = binding.producer_port.module_name + "." + binding.producer_port.port_name + serialized_outputs["outputs"].append(source + "->" + key) # Return the result. return serialized_outputs + + + def deserialize(self, serialized_outputs, modules): + """ + Method responsible for deserialization of graph outputs. + + Args: + serialized_outputs: A list of serialized outputs in the form of ("module.output_port->key") + modules: List of modules required for neural type copying/checking. + """ + # Check type. + if serialized_outputs["type"] == "default": + # We do not need to deserialize. + # self._default_outputs will be recorded automatically during graph execution. + # TODO: check neural types. + return + + # Iterate through serialized inputs one by one. + for i in serialized_outputs["outputs"]: + # Deserialize! + [producer, key] = i.split("->") + [producer_name, producer_port_name] = producer.split(".") + + # Get neural type from module output port definition. + n_type = modules[producer_name].output_ports[producer_port_name] + # Create a new input. + go = GraphOutput(n_type, ModulePort(producer_name, producer_port_name)) + self._manual_outputs[key] = go + # TODO: check neural types. + + # Done. + + diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 0fdaa0685456..c055ee4c3a1f 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -144,6 +144,7 @@ def nest(self, inner_graph, inner_graph_args): for t in tensors.values(): inner_connections.extend(t.connections()) + # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce # them only for ports that are already indicated as the "bound" ones in the inner graph. self.default_output_binding = False @@ -258,8 +259,11 @@ def bind_outputs(self, tensors_list): """ Binds the output tensors. Args: - tensors_list: List of tensors to be bound. + tensors_list: A single tensor OR a List of tensors to be bound. """ + # Handle both single port and lists of ports to be bound. + if type(tensors_list) is not list: + tensors_list = [tensors_list] # Add tensors list to of tensors. for tensor in tensors_list: # Add tensor to "all" tensors dictionary. @@ -495,7 +499,7 @@ def __serialize_connections(self): for c in tensor.connections(): # Serialize! source = c.producer.module_name + "." + c.producer.port_name - target = c.consumer.module_name + "." + c.producer.port_name + target = c.consumer.module_name + "." + c.consumer.port_name serialized_connections.append(source + "->" + target) return serialized_connections @@ -561,14 +565,16 @@ def deserialize(cls, configuration, reuse_existing_modules=False, name=None): steps = new_graph.__deserialize_steps(configuration["steps"]) # Deserialize the connections between modules. - connnections = new_graph.__deserialize_connections(configuration["connections"]) + connections = new_graph.__deserialize_connections(configuration["connections"]) - # TODO Deserialize input bindings. - # new_graph.__deserialize_inputs(configuration["inputs"]) + # Deserialize input bindings - return it in an external entity. + inputs = GraphInputs.deserialize(configuration["inputs"], modules) - # Now we have to execute the graph, following the steps and connections. + # Deserialize "manual" output bindings. + new_graph._outputs.deserialize(configuration["outputs"], modules) - # TODO: deserialize output bindings. + # Now we have to execute the graph, following the steps and connections. + new_graph.__execute_and_create_tensors(steps, modules, connections, inputs) # Return the graph instance. return new_graph @@ -589,7 +595,7 @@ def __deserialize_header(cls, serialized_header): # Get operation mode. if serialized_header["operation_mode"] == "training": operation_mode = OperationMode.training - elif header["operation_mode"] == "inference": + elif serialized_header["operation_mode"] == "inference": operation_mode = OperationMode.inference else: operation_mode = OperationMode.both @@ -660,14 +666,22 @@ def __deserialize_connections(self, serialized_connections): return connections def __execute_and_create_tensors(self, steps, modules, connections, inputs): - """ Method creates (internal) tensors of the graph by executing it following the order. """ + """ Method creates (internal) tensors of the graph by executing it following the order and using + the provided connections and inputs. + Args: + steps: + modules + connections + inputs + + """ # Activate this graph, so all the tensors will be added to this ! self.activate() # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce # them only for ports that are already indicated as the "bound" ones in the inner graph. - self.default_output_binding = False + # self.default_output_binding = False # Now "copy" graph execution order and topology by actually executing each step of the nested graph. for _, module_name in steps.items(): @@ -686,38 +700,44 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # Check if this port was bound in the inner graph. key = inputs.has_binding(module_name, input_port_name) + #import pdb;pdb.set_trace() # If so, then we must pass the binding! if key is not None: - module_args[input_port_name] = new_graph.inputs[key] - continue + # Copy the port "definition" (i.e. is NeuralType) using the same port name. + self.inputs[key] = inputs[key] + + # Pass this object to module input argument. + module_args[input_port_name] = self.inputs[key] # Else: find a tensor that should be passed to the given module's input. - # Search for producer/port that we should use. - for connection in connections: - if ( - connection.consumer.module_name == module_name - and connection.consumer.port_name == input_port_name - ): - # Got the connection! - producer_name = connection.producer.module_name - producer_port_name = connection.producer.port_name - break - # Now, the tensor is already produced in outer (i.e. this) graph! - module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + else: + # Search for producer/port that we should use. + for connection in connections: + if ( + connection.consumer.module_name == module_name + and connection.consumer.port_name == input_port_name + ): + # Got the connection! + producer_name = connection.producer.module_name + producer_port_name = connection.producer.port_name + break + # Now, the tensor is already produced in outer (i.e. this) graph! + module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + # End: for - # import pdb;pdb.set_trace() # Ok, now we have all keyword arguments. We can call() the module. # This will collect all the produced output tensors and add them to this graph. module(**module_args) - def __deserialize_inputs(self, serialized_inputs): - """ Private method deserializing the bound graph inputs. + # At that point we have all modules, steps and tensors added to outer (self) graph. + # Now we have to prepare the outputs. + + # Deactivate graph. + self.deactivate() + + # Ok, now we can turn automatic binding on. + #self.default_output_binding = True - Args: - serialized_steps: Dictionary containing serialized inputs. - """ - # TODO! - pass def __str__(self): """ Prints a nice summary. """ diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index aa70991f2248..b96fdbddb47d 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -567,7 +567,7 @@ def __call__(self, **kwargs): # Bind the neural graph input port, i.e. remember that a given graph port should pass data # to THIS module-port (when it finally will be connected). - active_graph.inputs[port_name].bind([ModulePort(self.name, port_name)]) + active_graph.inputs[port_name].bind(ModulePort(self.name, port_name)) # Please note that there are no "consumers" here - this is a "pure binding". @@ -589,7 +589,7 @@ def __call__(self, **kwargs): # Bind the neural graph input port, i.e. remember that a given graph port should pass data # to THIS module-port (when it finally will be connected). - port_content.bind([ModulePort(self.name, port_name)]) + port_content.bind(ModulePort(self.name, port_name)) # Please note that there are no "consumers" here - this is a "pure binding". @@ -616,7 +616,7 @@ def __call__(self, **kwargs): results = NmTensor(producer=self, producer_args=kwargs, output_port_name=out_name, ntype=out_type,) # Bind the "default" output ports. - self._app_state.active_graph.bind_outputs([results]) + self._app_state.active_graph.bind_outputs(results) else: # Create output tensors. output_tensors = [] From 7b5b9d457942b6a65d8cf83d2b86acbce8314a04 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 14:15:44 -0700 Subject: [PATCH 058/106] merged with master and reformatted Signed-off-by: Tomasz Kornuta --- ...raph_composition_integration_tests0_jasper_named.py | 2 +- .../graph_composition_integration_tests2_1.py | 8 ++++---- .../graph_composition_integration_tests3_1.py | 8 ++++---- nemo/core/neural_graph/graph_inputs.py | 2 +- nemo/core/neural_graph/graph_outputs.py | 3 --- nemo/core/neural_graph/neural_graph.py | 10 ++++------ 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py index c5a98a9d3aae..43e6449be936 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -76,7 +76,7 @@ serialized_jasper = Jasper.serialize() print("Serialized:\n", serialized_jasper) -# 'connections': +# 'connections': # * ['module1.processed_signal->module2.processed_signal', # * 'module1.processed_length->module2.processed_length', # * 'module2.outputs->module3.outputs'], diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index 860ee1cb8179..9fd3d5e90204 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -49,10 +49,10 @@ print("Serialized:\n", serialized_g1) # Delete everything! -#del g1 -#del dl -#del m1 -#del m2 +# del g1 +# del dl +# del m1 +# del m2 # Deserialize graph. g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=True, name="g1_copy") diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py index ac074651543c..c7f3375942f7 100644 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ b/examples/start_here/graph_composition_integration_tests3_1.py @@ -48,8 +48,8 @@ print("Serialized:\n", serialized_model) # Delete everything! -#del model -#del fx +# del model +# del fx # Deserialize graph. model_copy = NeuralGraph.deserialize(serialized_model, reuse_existing_modules=True, name="model_copy") @@ -74,5 +74,5 @@ nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") # Serialize graph -#print(model.serialize()) -#print(training.serialize()) +# print(model.serialize()) +# print(training.serialize()) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 69ce4611fb86..48c890263ad7 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -178,4 +178,4 @@ def deserialize(cls, serialized_inputs, modules, definitions_only=False): # Bound input. inputs[key].bind(ModulePort(consumer_name, consumer_port_name)) # Done. - return inputs \ No newline at end of file + return inputs diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index cccc69bd6440..1cc0e3de8a18 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -188,7 +188,6 @@ def serialize(self): # Return the result. return serialized_outputs - def deserialize(self, serialized_outputs, modules): """ Method responsible for deserialization of graph outputs. @@ -218,5 +217,3 @@ def deserialize(self, serialized_outputs, modules): # TODO: check neural types. # Done. - - diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index c055ee4c3a1f..6d0895378023 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -144,7 +144,6 @@ def nest(self, inner_graph, inner_graph_args): for t in tensors.values(): inner_connections.extend(t.connections()) - # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce # them only for ports that are already indicated as the "bound" ones in the inner graph. self.default_output_binding = False @@ -700,17 +699,17 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # Check if this port was bound in the inner graph. key = inputs.has_binding(module_name, input_port_name) - #import pdb;pdb.set_trace() + # import pdb;pdb.set_trace() # If so, then we must pass the binding! if key is not None: # Copy the port "definition" (i.e. is NeuralType) using the same port name. self.inputs[key] = inputs[key] - + # Pass this object to module input argument. module_args[input_port_name] = self.inputs[key] # Else: find a tensor that should be passed to the given module's input. - else: + else: # Search for producer/port that we should use. for connection in connections: if ( @@ -736,8 +735,7 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): self.deactivate() # Ok, now we can turn automatic binding on. - #self.default_output_binding = True - + # self.default_output_binding = True def __str__(self): """ Prints a nice summary. """ From 26a762187dbeca1e669a5372354033044802465e Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:02:30 -0700 Subject: [PATCH 059/106] Updated EN and ZH:] version of documentation for configuration/custom configuration Signed-off-by: Tomasz Kornuta --- .../source/tutorials/module_configuration.rst | 8 +++--- .../tutorials/module_custom_configuration.rst | 23 ++++++--------- .../source/tutorials/module_configuration.rst | 10 +++---- .../tutorials/module_custom_configuration.rst | 28 ++++++++----------- 4 files changed, 28 insertions(+), 41 deletions(-) diff --git a/docs/docs_zh/sources/source/tutorials/module_configuration.rst b/docs/docs_zh/sources/source/tutorials/module_configuration.rst index 6780d897f4ae..fcfeaddd3183 100644 --- a/docs/docs_zh/sources/source/tutorials/module_configuration.rst +++ b/docs/docs_zh/sources/source/tutorials/module_configuration.rst @@ -21,14 +21,14 @@ .. literalinclude:: ../../../../../examples/start_here/module_configuration.py :language: python - :lines: 25-35 + :lines: 24-34 现在我们可以导出任何一个已有模块的配置，调用 :meth:`export_to_config()`, 例如 \ 我们可以导出 :class:`TaylorNet` 的配置，通过调用: .. literalinclude:: ../../../../../examples/start_here/module_configuration.py :language: python - :lines: 38 + :lines: 37 导入配置 --------- @@ -37,7 +37,7 @@ .. literalinclude:: ../../../../../examples/start_here/module_configuration.py :language: python - :lines: 41 + :lines: 40 .. note:: :meth:`import_from_config()` 函数事实上是创建了在配置中的这个类的一个新的实例 \ @@ -49,7 +49,7 @@ .. literalinclude:: ../../../../../examples/start_here/module_configuration.py :language: python - :lines: 43- + :lines: 42- .. include:: module_custom_configuration.rst diff --git a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst index 39f7db0540f1..b20b285a4a08 100644 --- a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst @@ -15,13 +15,13 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 33-35 + :lines: 31-33 现在让我们定义 :class:`CustomTaylorNet` 神经模块类: .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 38-43 + :lines: 36-41 为了能处理好 :class:`Status` enum 的导出功能，我们必须实现自定义函数 \ @@ -29,7 +29,7 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 45-76 + :lines: 43-64 注意配置实际上是一个字典，包含了两个部分: @@ -40,21 +40,14 @@ 这些参数存在保护域 ``self._init_params`` 中，它的基类是 :class:`NeuralModule` 类。 确保用户不能直接访问和使用它们。 -类似地，我们必须重载方法 :meth:`import_from_config()` : +类似地，我们必须重载方法 :meth:`_deserialize_configuration()` : .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 79-119 - -请注意，基类 :class:`NeuralModule` 提供了一些保护方法供我们使用, \ -其中，最重要的是: - - * :meth:`__serialize_header()` 生成合适的 header, 以及 \ - * :meth:`__validate_config_file()` 验证加载的配置文件 (检查 header 内容)。 - + :lines: 66-89 .. note:: - 再强调一下 :meth:`import_from_config()` 是类的方法，实际上返回 \ + 再强调一下 :meth:`_deserialize_configuration()` 是类的方法，实际上返回 \ 一个新的对象实例 - 在这个例子中就是 :class:`CustomTaylorNet` 类型。 @@ -62,13 +55,13 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 128-129,134-135 + :lines: 98-99,104-105 通过加载这个配置，初始化第二个实例: .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 137-139 + :lines: 107-109 从结果中我们可以看到新的对象把状态都设置成了原来那个对象的值: diff --git a/docs/sources/source/tutorials/module_configuration.rst b/docs/sources/source/tutorials/module_configuration.rst index 08f998a9d8fe..b009e3edca75 100644 --- a/docs/sources/source/tutorials/module_configuration.rst +++ b/docs/sources/source/tutorials/module_configuration.rst @@ -17,18 +17,18 @@ In the following example we will once again train a model to learn Taylor's coef However, we will extend the example by showing how to export configuration of the module to a YAML file and \ create a second instance having the same set of parameters. -Let us start by creating the :class:`NeuralFactory` object and instatiating the modules from the original example: +Let us start by creating the :class:`NeuralModuleFactory` object and instatiating the modules from the original example: .. literalinclude:: ../../../../examples/start_here/module_configuration.py :language: python - :lines: 25-35 + :lines: 24-34 Now we can export the configuration of any of the existing modules by using the :meth:`export_to_config()`, for \ example we can export the configuration of the trainable :class:`TaylorNet` by calling: .. literalinclude:: ../../../../examples/start_here/module_configuration.py :language: python - :lines: 38 + :lines: 37 Importing the configuration --------------------------- @@ -37,7 +37,7 @@ There is an analogical function :meth:`import_from_config()` responsible for loa .. literalinclude:: ../../../../examples/start_here/module_configuration.py :language: python - :lines: 41 + :lines: 40 .. note:: The :meth:`import_from_config()` function actually creates a new instance of object of the class that was stored \ @@ -49,7 +49,7 @@ For example, we can build a graph and train it with a NeMo trainer: .. literalinclude:: ../../../../examples/start_here/module_configuration.py :language: python - :lines: 43- + :lines: 42- .. include:: module_custom_configuration.rst diff --git a/docs/sources/source/tutorials/module_custom_configuration.rst b/docs/sources/source/tutorials/module_custom_configuration.rst index 10c12bd5a45a..9229bdf30149 100644 --- a/docs/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/sources/source/tutorials/module_custom_configuration.rst @@ -15,21 +15,21 @@ and extend it by those methods. But first, let us define a simple :class:`Status .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 33-35 + :lines: 31-33 Now let us define the :class:`CustomTaylorNet` Neural Module class: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 38-43 + :lines: 36-41 In order to properly handle the export of the :class:`Status` enum we must implement a custom function \ -:meth:`export_to_config()`: +:meth:`_serialize_configuration()`: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 45-76 + :lines: 43-64 Note that the configuration is actually a dictionary consisting of two sections: @@ -40,35 +40,29 @@ Note that the configuration is actually a dictionary consisting of two sections: Those parameters are stored in the protected ``self._init_params`` field of the base :class:`NeuralModule` class. It is assumed that (aside of this use-case) the user won't access nor use them directly. -Analogically, we must overload the :meth:`import_from_config()` method: +Analogically, we must overload the :meth:`_deserialize_configuration()` method: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 79-119 - -Please note that the base :class:`NeuralModule` class provides several protected methods that we used, \ -with most important being: - - * :meth:`__serialize_header()` generating the appropriate header, and \ - * :meth:`__validate_config_file()` validating the loaded configuration file (checking the header content). - + :lines: 66-89 .. note:: - It is once again worth emphasizing that the :meth:`import_from_config()` is a class method, actually returning a \ - new object instance - in this case of the hardcoded :class:`CustomTaylorNet` type. + It is worth emphasizing that the :meth:`_deserialize_configuration()` is a class method, + analogically to public :meth:`import_from_config()` and :meth:`deserialize()` methods + that return a new object instance - in this case of the hardcoded :class:`CustomTaylorNet` type. Now we can simply create an instance and export its configuration by calling: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 128-129,134-135 + :lines: 98-99,104-105 And instantiate a second by loading that configuration: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 137-139 + :lines: 107-109 As a result we will see that the new object has set the status to the same value as the original one: From 314cf1029b6d0619427d17b7fec36effe783d232 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:12:34 -0700 Subject: [PATCH 060/106] Unification of singleton meta class Signed-off-by: Tomasz Kornuta --- nemo/utils/app_state.py | 26 +------------------------- nemo/utils/metaclasses.py | 29 +++++++++++++++++++++-------- nemo/utils/nemo_logging.py | 4 ++-- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index 23ae2783cf18..db7fc6ff48de 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -15,33 +15,9 @@ # limitations under the License. # ============================================================================= -import threading - # Sadly have to import this to avoid circular dependencies. import nemo - - -class Singleton(type): - """ Implementation of a generic, tread-safe singleton meta-class. - Can be used as meta-class, i.e. will create - """ - - # List of instances - one per class. - __instances = {} - # Lock used for accessing the instance. - __lock = threading.Lock() - - def __call__(cls, *args, **kwargs): - """ Returns singleton instance.A thread safe implementation. """ - if cls not in cls.__instances: - # Enter critical section. - with cls.__lock: - # Check once again. - if cls not in cls.__instances: - # Create a new object instance - one per class. - cls.__instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - # Return the instance. - return cls.__instances[cls] +from nemo.utils.metaclasses import Singleton class AppState(metaclass=Singleton): diff --git a/nemo/utils/metaclasses.py b/nemo/utils/metaclasses.py index 0f584aa76cad..8aed2d240e00 100644 --- a/nemo/utils/metaclasses.py +++ b/nemo/utils/metaclasses.py @@ -13,17 +13,30 @@ # limitations under the License.**** __all__ = [ - "SingletonMetaClass", + "Singleton", ] +import threading -class SingletonMetaClass(type): - _instances = {} +class Singleton(type): + """ Implementation of a generic, tread-safe singleton meta-class. + Can be used as meta-class, i.e. will create + """ - def __call__(cls, *args, **kwargs): - - if cls not in cls._instances: - cls._instances[cls] = super(SingletonMetaClass, cls).__call__(*args, **kwargs) + # List of instances - one per class. + __instances = {} + # Lock used for accessing the instance. + __lock = threading.Lock() - return cls._instances[cls] + def __call__(cls, *args, **kwargs): + """ Returns singleton instance.A thread safe implementation. """ + if cls not in cls.__instances: + # Enter critical section. + with cls.__lock: + # Check once again. + if cls not in cls.__instances: + # Create a new object instance - one per class. + cls.__instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + # Return the instance. + return cls.__instances[cls] diff --git a/nemo/utils/nemo_logging.py b/nemo/utils/nemo_logging.py index cc86ba66a6a7..1551acf84839 100644 --- a/nemo/utils/nemo_logging.py +++ b/nemo/utils/nemo_logging.py @@ -23,7 +23,7 @@ from nemo.constants import NEMO_ENV_VARNAME_REDIRECT_LOGS_TO_STDERR from nemo.utils.env_var_parsing import get_envbool, get_envint from nemo.utils.formatters.base import BaseNeMoFormatter -from nemo.utils.metaclasses import SingletonMetaClass +from nemo.utils.metaclasses import Singleton __all__ = ["Logger", "LogMode"] @@ -33,7 +33,7 @@ class LogMode(enum.IntEnum): ONCE = 1 # Log the message only once. The same message will not be logged again. -class Logger(metaclass=SingletonMetaClass): +class Logger(metaclass=Singleton): # Level 0 NOTSET = _logging.NOTSET From 7f0ee860294ea1ceb362952fafb32dc4fd2b4ad0 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:30:28 -0700 Subject: [PATCH 061/106] Added to the train() function signatures Signed-off-by: Tomasz Kornuta --- ...osition_integration_tests0_jasper_named.py | 8 +++-- nemo/backends/pytorch/actions.py | 15 +++++++- nemo/core/neural_factory.py | 4 ++- nemo/core/neural_graph/graph_outputs.py | 34 +++++++++++++++++-- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py index 43e6449be936..a811965a9a9d 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -102,7 +102,7 @@ print("Deserialized:\n", serialized_jasper_copy) assert serialized_jasper == serialized_jasper_copy -with NeuralGraph(name="training") as training: +with NeuralGraph(name="training") as training_graph: # Create the "implicit" training graph. o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() # Use Jasper module as any other neural module. @@ -114,13 +114,17 @@ log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len ) tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] + # Set graph output. + training_graph.outputs["o_loss"] = o_loss + # training_graph.outputs["o_predictions"] = o_predictions # DOESN'T WORK?!? train_callback = nemo.core.SimpleLossLoggerCallback( tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) ) # import pdb;pdb.set_trace() nf.train( - tensors_to_optimize=[o_loss], + #tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? + training_graph=training_graph, optimizer="novograd", callbacks=[train_callback], optimization_params={"num_epochs": 50, "lr": 0.01}, diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 68f7c6496d22..a9037c95d21b 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -1087,7 +1087,8 @@ def _check_nan_or_inf(self, placement_gpu, nan_or_inf, steps_per_nan_check=None) def train( self, - tensors_to_optimize, + tensors_to_optimize=None, + training_graph=None, optimizer=None, optimization_params=None, callbacks: Optional[List[ActionCallback]] = None, @@ -1100,6 +1101,18 @@ def train( gradient_predivide=False, amp_max_loss_scale=2.0 ** 24, ): + # Analyse the arguments passed to train. + if tensors_to_optimize is None and training_graph is None: + raise ValueError("Cannot pass both `tensors_to_optimize` and `training_graph` to the train() function") + if tensors_to_optimize is not None and training_graph is not None: + raise ValueError( + "One of the `tensors_to_optimize` or `training_graph` values must be passed to the train() function" + ) + # Finally, unify. + if training_graph is not None: + # To keep the "compatibility with old NeMo": get output tensors. + tensors_to_optimize = training_graph.outputs.tensor_list + if gradient_predivide: logging.error( "gradient_predivide is currently disabled, and is under consideration for removal in future versions. " diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 3e6a8f9f1290..8aecda3d64d9 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -565,7 +565,8 @@ def create_optimizer(self, optimizer, things_to_optimize, optimizer_params): def train( self, - tensors_to_optimize, + tensors_to_optimize=None, + training_graph=None, optimizer=None, optimization_params=None, callbacks: Optional[List[ActionCallback]] = None, @@ -583,6 +584,7 @@ def train( self.reset_trainer() return self._trainer.train( tensors_to_optimize=tensors_to_optimize, + training_graph=training_graph, optimizer=optimizer, optimization_params=optimization_params, callbacks=callbacks, diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 1cc0e3de8a18..176562086e68 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -148,7 +148,12 @@ def definitions(self): @property def tensors(self): - """ Property returns output tensors by extracting them on the fly from the bound outputs. """ + """ + Property returns output tensors by extracting them on the fly from the bound outputs. + + Returns: + Dictionary of tensors in the format (output-name: tensor). + """ # Get the right output dictionary. d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs @@ -161,9 +166,34 @@ def tensors(self): tensor = self._tensors_list[producer_name][producer_port_name] # Add it to the dictionary. output_tensors[k] = tensor - + # Return the result. return output_tensors + @property + def tensor_list(self): + """ + Property returns output tensors by extracting them on the fly from the bound outputs. + + Returns: + List of tensors. + + """ + # Get the right output dictionary. + d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs + + output_tensor_list = [] + # Get tensors by acessing the producer-ports. + for k, v in d.items(): + producer_name = v.producer_port.module_name + producer_port_name = v.producer_port.port_name + # Find the right output tensor. + tensor = self._tensors_list[producer_name][producer_port_name] + # Add it to the list. + output_tensor_list.append(tensor) + # Return the result. + return output_tensor_list + + def serialize(self): """ Method responsible for serialization of the graph outputs. From 40768089fefe7a5343607456b78b514b781c3343 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:31:23 -0700 Subject: [PATCH 062/106] formatting fix Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0_jasper_named.py | 2 +- nemo/core/neural_graph/graph_outputs.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py index a811965a9a9d..679ae44d6acc 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper_named.py @@ -123,7 +123,7 @@ ) # import pdb;pdb.set_trace() nf.train( - #tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? + # tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? training_graph=training_graph, optimizer="novograd", callbacks=[train_callback], diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 176562086e68..98c2b713ca77 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -193,7 +193,6 @@ def tensor_list(self): # Return the result. return output_tensor_list - def serialize(self): """ Method responsible for serialization of the graph outputs. From 41bb68f82c01dc1f2dfd79762d77e568069198a4 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:33:46 -0700 Subject: [PATCH 063/106] LGTM fixes Signed-off-by: Tomasz Kornuta --- examples/start_here/graph_composition_integration_tests2_1.py | 2 -- examples/start_here/module_custom_configuration.py | 3 --- nemo/core/neural_graph/graph_inputs.py | 1 - 3 files changed, 6 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index 9fd3d5e90204..4e88ddf463e0 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -17,8 +17,6 @@ # limitations under the License. # ============================================================================= -import pdb - from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback from nemo.utils import logging diff --git a/examples/start_here/module_custom_configuration.py b/examples/start_here/module_custom_configuration.py index 793094b12e70..4f406304de23 100644 --- a/examples/start_here/module_custom_configuration.py +++ b/examples/start_here/module_custom_configuration.py @@ -18,9 +18,6 @@ # ============================================================================= from enum import Enum -from os import path - -from ruamel import yaml from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralModuleFactory, SimpleLossLoggerCallback diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 48c890263ad7..85f2cace3be6 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -16,7 +16,6 @@ # limitations under the License. # ============================================================================= -from collections import namedtuple from collections.abc import MutableMapping from nemo.core.neural_types import NeuralType From cb56b359a55a74611c095b7da629e17f81a79fc3 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:35:35 -0700 Subject: [PATCH 064/106] line numbers Signed-off-by: Tomasz Kornuta --- .../source/tutorials/module_custom_configuration.rst | 12 ++++++------ .../source/tutorials/module_custom_configuration.rst | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst index b20b285a4a08..d9834a1d2dd4 100644 --- a/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/docs_zh/sources/source/tutorials/module_custom_configuration.rst @@ -15,13 +15,13 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 31-33 + :lines: 28-30 现在让我们定义 :class:`CustomTaylorNet` 神经模块类: .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 36-41 + :lines: 33-38 为了能处理好 :class:`Status` enum 的导出功能，我们必须实现自定义函数 \ @@ -29,7 +29,7 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 43-64 + :lines: 40-61 注意配置实际上是一个字典，包含了两个部分: @@ -44,7 +44,7 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 66-89 + :lines: 63-86 .. note:: 再强调一下 :meth:`_deserialize_configuration()` 是类的方法，实际上返回 \ @@ -55,13 +55,13 @@ .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 98-99,104-105 + :lines: 95-96,101-102 通过加载这个配置，初始化第二个实例: .. literalinclude:: ../../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 107-109 + :lines: 104-106 从结果中我们可以看到新的对象把状态都设置成了原来那个对象的值: diff --git a/docs/sources/source/tutorials/module_custom_configuration.rst b/docs/sources/source/tutorials/module_custom_configuration.rst index 9229bdf30149..8c368ea0b4b1 100644 --- a/docs/sources/source/tutorials/module_custom_configuration.rst +++ b/docs/sources/source/tutorials/module_custom_configuration.rst @@ -15,13 +15,13 @@ and extend it by those methods. But first, let us define a simple :class:`Status .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 31-33 + :lines: 28-30 Now let us define the :class:`CustomTaylorNet` Neural Module class: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 36-41 + :lines: 33-38 In order to properly handle the export of the :class:`Status` enum we must implement a custom function \ @@ -29,7 +29,7 @@ In order to properly handle the export of the :class:`Status` enum we must imple .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 43-64 + :lines: 49-61 Note that the configuration is actually a dictionary consisting of two sections: @@ -44,7 +44,7 @@ Analogically, we must overload the :meth:`_deserialize_configuration()` method: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 66-89 + :lines: 63-86 .. note:: It is worth emphasizing that the :meth:`_deserialize_configuration()` is a class method, @@ -56,13 +56,13 @@ Now we can simply create an instance and export its configuration by calling: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 98-99,104-105 + :lines: 95-96,101-102 And instantiate a second by loading that configuration: .. literalinclude:: ../../../../examples/start_here/module_custom_configuration.py :language: python - :lines: 107-109 + :lines: 104-106 As a result we will see that the new object has set the status to the same value as the original one: From 5ff38964c9edc177ae88e3b9e792f9d0f6401295 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:58:38 -0700 Subject: [PATCH 065/106] Tweak of deserialize - moved name and overwrite_params Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 96 +++++++++---- ...osition_integration_tests0_jasper_named.py | 131 ------------------ nemo/core/neural_modules.py | 32 +++-- 3 files changed, 86 insertions(+), 173 deletions(-) delete mode 100644 examples/start_here/graph_composition_integration_tests0_jasper_named.py diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 0fab64908815..ddbd2e2a31a9 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -25,14 +25,15 @@ import nemo import nemo.collections.asr as nemo_asr from nemo.collections.asr.helpers import monitor_asr_train_progress -from nemo.core import NeuralGraph - -logging = nemo.logging +from nemo.core import NeuralGraph, OperationMode +from nemo.utils import logging +from nemo.utils.app_state import AppState nf = nemo.core.NeuralModuleFactory() +app_state = AppState() logging.info( - "This example shows how one can build a Jasper model using the `default` (implicit) graph." + "This example shows how one can build a Jasper model using the explicit graph." F" This approach works for applications containing a single graph." ) @@ -43,48 +44,83 @@ yaml = YAML(typ="safe") with open(expanduser(model_config_file)) as f: - jasper_params = yaml.load(f) + config = yaml.load(f) # Get vocabulary. -vocab = jasper_params['labels'] +vocab = config['labels'] # Create neural modules. -data_layer = nemo_asr.AudioToTextDataLayer.import_from_config( - model_config_file, - "AudioToTextDataLayer_train", +data_layer = nemo_asr.AudioToTextDataLayer.deserialize( + config["AudioToTextDataLayer_train"], overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, ) -data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.import_from_config( - model_config_file, "AudioToMelSpectrogramPreprocessor" -) +data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.deserialize(config["AudioToMelSpectrogramPreprocessor"]) -jasper_encoder = nemo_asr.JasperEncoder.import_from_config(model_config_file, "JasperEncoder") -jasper_decoder = nemo_asr.JasperDecoderForCTC.import_from_config( - model_config_file, "JasperDecoderForCTC", overwrite_params={"num_classes": len(vocab)} +jasper_encoder = nemo_asr.JasperEncoder.deserialize(config["JasperEncoder"]) +jasper_decoder = nemo_asr.JasperDecoderForCTC.deserialize(config["JasperDecoderForCTC"], overwrite_params={"num_classes": len(vocab)} ) ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) greedy_decoder = nemo_asr.GreedyCTCDecoder() # Create the Jasper composite module. -with NeuralGraph() as Jasper: - processed_signal, processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. - encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=processed_signal_len) - _ = jasper_decoder(encoder_output=encoded) # All output ports are bind (for now!) - -# Create the "implicit" training graph. -audio_signal, audio_signal_len, transcript, transcript_len = data_layer() -# Use Jasper module as any other neural module. -_, _, _, encoded_len, log_probs = Jasper(input_signal=audio_signal, length=audio_signal_len) -predictions = greedy_decoder(log_probs=log_probs) -loss = ctc_loss(log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len) -tensors_to_evaluate = [loss, predictions, transcript, transcript_len] - +with NeuralGraph(operation_mode=OperationMode.both) as Jasper: + i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. + i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) + i_log_probs = jasper_decoder(encoder_output=i_encoded) # All output ports are bind (for now!) + +# Serialize graph +serialized_jasper = Jasper.serialize() +print("Serialized:\n", serialized_jasper) + +# 'connections': +# * ['module1.processed_signal->module2.processed_signal', +# * 'module1.processed_length->module2.processed_length', +# * 'module2.outputs->module3.outputs'], +# 'inputs': +# * ['input_signal->module1.input_signal', +# * 'length->module1.length'], +# 'outputs': +# * {'outputs': ['module1.processed_signal->processed_signal', +# * 'module1.processed_length->processed_length', +# * 'module2.outputs->outputs', +# * 'module2.encoded_lengths->encoded_lengths', +# * 'module3.output->output'] + +# Delete everything - aside of jasper encoder, just as a test! ;) +del Jasper +del data_preprocessor +# del jasper_encoder # +del jasper_decoder + +# Deserialize graph. +jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") +serialized_jasper_copy = jasper_copy.serialize() +print("Deserialized:\n", serialized_jasper_copy) +assert serialized_jasper == serialized_jasper_copy + +with NeuralGraph(name="training") as training_graph: + # Create the "implicit" training graph. + o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() + # Use Jasper module as any other neural module. + o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = jasper_copy( + input_signal=o_audio_signal, length=o_audio_signal_len + ) + o_predictions = greedy_decoder(log_probs=o_log_probs) + o_loss = ctc_loss( + log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len + ) + # Set graph output. + training_graph.outputs["o_loss"] = o_loss + # training_graph.outputs["o_predictions"] = o_predictions # DOESN'T WORK?!? + +tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] train_callback = nemo.core.SimpleLossLoggerCallback( tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) ) - +# import pdb;pdb.set_trace() nf.train( - tensors_to_optimize=[loss], + # tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? + training_graph=training_graph, optimizer="novograd", callbacks=[train_callback], optimization_params={"num_epochs": 50, "lr": 0.01}, diff --git a/examples/start_here/graph_composition_integration_tests0_jasper_named.py b/examples/start_here/graph_composition_integration_tests0_jasper_named.py deleted file mode 100644 index 679ae44d6acc..000000000000 --- a/examples/start_here/graph_composition_integration_tests0_jasper_named.py +++ /dev/null @@ -1,131 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from functools import partial -from os.path import expanduser - -from ruamel.yaml import YAML - -import nemo -import nemo.collections.asr as nemo_asr -from nemo.collections.asr.helpers import monitor_asr_train_progress -from nemo.core import NeuralGraph, OperationMode -from nemo.utils import logging -from nemo.utils.app_state import AppState - -nf = nemo.core.NeuralModuleFactory() -app_state = AppState() - -logging.info( - "This example shows how one can build a Jasper model using the `default` (implicit) graph." - F" This approach works for applications containing a single graph." -) - -# Set paths to "manifests" and model configuration files. -train_manifest = "~/TestData/an4_dataset/an4_train.json" -val_manifest = "~/TestData/an4_dataset/an4_val.json" -model_config_file = "~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" - -yaml = YAML(typ="safe") -with open(expanduser(model_config_file)) as f: - jasper_params = yaml.load(f) -# Get vocabulary. -vocab = jasper_params['labels'] - -# Create neural modules. -data_layer = nemo_asr.AudioToTextDataLayer.import_from_config( - model_config_file, - "AudioToTextDataLayer_train", - overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, -) - -data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.import_from_config( - model_config_file, "AudioToMelSpectrogramPreprocessor" -) - -jasper_encoder = nemo_asr.JasperEncoder.import_from_config(model_config_file, "JasperEncoder") -jasper_decoder = nemo_asr.JasperDecoderForCTC.import_from_config( - model_config_file, "JasperDecoderForCTC", overwrite_params={"num_classes": len(vocab)} -) -ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) -greedy_decoder = nemo_asr.GreedyCTCDecoder() - -# Create the Jasper composite module. -with NeuralGraph(operation_mode=OperationMode.both) as Jasper: - i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. - i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) - i_log_probs = jasper_decoder(encoder_output=i_encoded) # All output ports are bind (for now!) - -# Serialize graph -serialized_jasper = Jasper.serialize() -print("Serialized:\n", serialized_jasper) - -# 'connections': -# * ['module1.processed_signal->module2.processed_signal', -# * 'module1.processed_length->module2.processed_length', -# * 'module2.outputs->module3.outputs'], -# 'inputs': -# * ['input_signal->module1.input_signal', -# * 'length->module1.length'], -# 'outputs': -# * {'outputs': ['module1.processed_signal->processed_signal', -# * 'module1.processed_length->processed_length', -# * 'module2.outputs->outputs', -# * 'module2.encoded_lengths->encoded_lengths', -# * 'module3.output->output'] - -# Delete everything! -del Jasper -del jasper_encoder -del jasper_decoder -del data_preprocessor - -# Deserialize graph. -jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") -serialized_jasper_copy = jasper_copy.serialize() -print("Deserialized:\n", serialized_jasper_copy) -assert serialized_jasper == serialized_jasper_copy - -with NeuralGraph(name="training") as training_graph: - # Create the "implicit" training graph. - o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() - # Use Jasper module as any other neural module. - o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = jasper_copy( - input_signal=o_audio_signal, length=o_audio_signal_len - ) - o_predictions = greedy_decoder(log_probs=o_log_probs) - o_loss = ctc_loss( - log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len - ) - tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] - # Set graph output. - training_graph.outputs["o_loss"] = o_loss - # training_graph.outputs["o_predictions"] = o_predictions # DOESN'T WORK?!? - -train_callback = nemo.core.SimpleLossLoggerCallback( - tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) -) -# import pdb;pdb.set_trace() -nf.train( - # tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? - training_graph=training_graph, - optimizer="novograd", - callbacks=[train_callback], - optimization_params={"num_epochs": 50, "lr": 0.01}, -) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index b96fdbddb47d..764cf7917d2f 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -303,7 +303,9 @@ def import_from_config(cls, config_file, section_name=None, name=None, overwrite section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) - overwrite_init_params: Dictionary containing parameters that will be added to or overwrite (!) + name: name of the module that will overwrite the name in the `init_params` (optional, DEFAULT: None) + + overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) the default init parameters loaded from the configuration file (the module "init_params" section). Returns: @@ -313,34 +315,40 @@ def import_from_config(cls, config_file, section_name=None, name=None, overwrite # Validate the content of the configuration file (its header). loaded_config = cls.__validate_config_file(config_file, section_name) - # Update parameters with additional ones. - loaded_config["init_params"].update(overwrite_params) - - # Override module name in init_params using the logic: - # * section_name if not none overrides init_params.name first (skipped for now, TOTHINK!) - # * name (if None) overrides init_params.name - if name is not None: - loaded_config["init_params"]["name"] = name - # "Deserialize" the module. - obj = cls.deserialize(loaded_config) + obj = cls.deserialize(loaded_config, name, overwrite_params) return obj @classmethod - def deserialize(cls, configuration): + def deserialize(cls, configuration, name=None, overwrite_params={}): """ Class method instantianting the neural module object based on the configuration (dictionary). Args: configuration: Dictionary containing proper "header" and "init_params" sections. + name: name of the module that will overwrite the name in the `init_params` (optional, DEFAULT: None) + + overwrite_params: Dictionary containing parameters that will be added to or overwrite (!) + the default init parameters loaded from the configuration file (the module "init_params" section). + Returns: Instance of the created NeuralModule object. """ # Deserialize header - get object class. module_class = cls.__deserialize_header(configuration["header"]) + # Update parameters with additional ones. + configuration["init_params"].update(overwrite_params) + + # Override module name in init_params using the logic: + # * section_name if not none overrides init_params.name first (skipped for now, TOTHINK!) + # * name (if None) overrides init_params.name + if name is not None: + configuration["init_params"]["name"] = name + + # Get init parameters. init_params = cls._deserialize_configuration(configuration["init_params"]) From c955a7a4855d0ef82faabf791a5e394b6d68d4ce Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 17:59:44 -0700 Subject: [PATCH 066/106] style fix Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0_jasper.py | 8 ++++---- nemo/core/neural_modules.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index ddbd2e2a31a9..28be2eec9220 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -50,14 +50,14 @@ # Create neural modules. data_layer = nemo_asr.AudioToTextDataLayer.deserialize( - config["AudioToTextDataLayer_train"], - overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, + config["AudioToTextDataLayer_train"], overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, ) data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.deserialize(config["AudioToMelSpectrogramPreprocessor"]) jasper_encoder = nemo_asr.JasperEncoder.deserialize(config["JasperEncoder"]) -jasper_decoder = nemo_asr.JasperDecoderForCTC.deserialize(config["JasperDecoderForCTC"], overwrite_params={"num_classes": len(vocab)} +jasper_decoder = nemo_asr.JasperDecoderForCTC.deserialize( + config["JasperDecoderForCTC"], overwrite_params={"num_classes": len(vocab)} ) ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) greedy_decoder = nemo_asr.GreedyCTCDecoder() @@ -89,7 +89,7 @@ # Delete everything - aside of jasper encoder, just as a test! ;) del Jasper del data_preprocessor -# del jasper_encoder # +# del jasper_encoder # del jasper_decoder # Deserialize graph. diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 764cf7917d2f..dcb152d84617 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -348,7 +348,6 @@ def deserialize(cls, configuration, name=None, overwrite_params={}): if name is not None: configuration["init_params"]["name"] = name - # Get init parameters. init_params = cls._deserialize_configuration(configuration["init_params"]) From 3b47bd96ae790c42ea1aefa4d5838bae699b3a69 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 18:06:49 -0700 Subject: [PATCH 067/106] Commeting the condition for both None: eval() seems to be calling :] Signed-off-by: Tomasz Kornuta --- nemo/backends/pytorch/actions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index a9037c95d21b..ec24eb97362f 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -1102,12 +1102,12 @@ def train( amp_max_loss_scale=2.0 ** 24, ): # Analyse the arguments passed to train. - if tensors_to_optimize is None and training_graph is None: - raise ValueError("Cannot pass both `tensors_to_optimize` and `training_graph` to the train() function") if tensors_to_optimize is not None and training_graph is not None: - raise ValueError( - "One of the `tensors_to_optimize` or `training_graph` values must be passed to the train() function" - ) + raise ValueError("Cannot pass both `tensors_to_optimize` and `training_graph` to the train() function") + # if tensors_to_optimize is None and training_graph is None: + # raise ValueError( + # "One of the `tensors_to_optimize` or `training_graph` values must be passed to the train() function" + # ) # Finally, unify. if training_graph is not None: # To keep the "compatibility with old NeMo": get output tensors. From fd88c70a874aea476f45141c6fb934e9a88b41f9 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 18:09:21 -0700 Subject: [PATCH 068/106] LGTM fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/neural_graph_manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nemo/core/neural_graph/neural_graph_manager.py b/nemo/core/neural_graph/neural_graph_manager.py index edbca32eb4d5..1e128b34fb5d 100644 --- a/nemo/core/neural_graph/neural_graph_manager.py +++ b/nemo/core/neural_graph/neural_graph_manager.py @@ -29,6 +29,12 @@ def __init__(self): super().__init__("graph") self._active_graph = None + def __eq__(self, other): + """ Checks if two managers have the same content. """ + if not isinstance(other, ObjectRegistry): + return False + return super().__eq__(other) + def summary(self): """ Prints a nice summary. """ # TODO: a nicer summary. ;) From 98b03cf66f5f8ef29afb86a39d0c31942712edb5 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 27 Apr 2020 20:37:51 -0700 Subject: [PATCH 069/106] method comment Signed-off-by: Tomasz Kornuta --- nemo/utils/object_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 229bb70088d1..2d570aba82b1 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -101,7 +101,7 @@ def __getitem__(self, key): key: Object name. Returns: - Associated . + Object associated with the key. """ # Search for an object with a given name. for obj in self: From 18077fcfd308bc5a95a45a4e1d519d4f095afbc8 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 09:32:52 -0700 Subject: [PATCH 070/106] reorganization of directories in unit/core/utils Signed-off-by: Tomasz Kornuta --- tests/unit/core/{ => neural_graph}/test_graph_outputs_binding.py | 0 tests/unit/core/{ => neural_graph}/test_neural_graph_nesting.py | 0 tests/unit/core/{ => neural_graph}/test_neural_graphs.py | 0 tests/unit/core/{ => neural_module}/test_module_configuration.py | 0 .../core/{ => neural_module}/test_module_configuration_export.py | 0 .../core/{ => neural_module}/test_module_configuration_import.py | 0 tests/unit/core/{ => neural_module}/test_module_initialization.py | 0 tests/unit/{core => utils}/test_deprecated.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename tests/unit/core/{ => neural_graph}/test_graph_outputs_binding.py (100%) rename tests/unit/core/{ => neural_graph}/test_neural_graph_nesting.py (100%) rename tests/unit/core/{ => neural_graph}/test_neural_graphs.py (100%) rename tests/unit/core/{ => neural_module}/test_module_configuration.py (100%) rename tests/unit/core/{ => neural_module}/test_module_configuration_export.py (100%) rename tests/unit/core/{ => neural_module}/test_module_configuration_import.py (100%) rename tests/unit/core/{ => neural_module}/test_module_initialization.py (100%) rename tests/unit/{core => utils}/test_deprecated.py (100%) diff --git a/tests/unit/core/test_graph_outputs_binding.py b/tests/unit/core/neural_graph/test_graph_outputs_binding.py similarity index 100% rename from tests/unit/core/test_graph_outputs_binding.py rename to tests/unit/core/neural_graph/test_graph_outputs_binding.py diff --git a/tests/unit/core/test_neural_graph_nesting.py b/tests/unit/core/neural_graph/test_neural_graph_nesting.py similarity index 100% rename from tests/unit/core/test_neural_graph_nesting.py rename to tests/unit/core/neural_graph/test_neural_graph_nesting.py diff --git a/tests/unit/core/test_neural_graphs.py b/tests/unit/core/neural_graph/test_neural_graphs.py similarity index 100% rename from tests/unit/core/test_neural_graphs.py rename to tests/unit/core/neural_graph/test_neural_graphs.py diff --git a/tests/unit/core/test_module_configuration.py b/tests/unit/core/neural_module/test_module_configuration.py similarity index 100% rename from tests/unit/core/test_module_configuration.py rename to tests/unit/core/neural_module/test_module_configuration.py diff --git a/tests/unit/core/test_module_configuration_export.py b/tests/unit/core/neural_module/test_module_configuration_export.py similarity index 100% rename from tests/unit/core/test_module_configuration_export.py rename to tests/unit/core/neural_module/test_module_configuration_export.py diff --git a/tests/unit/core/test_module_configuration_import.py b/tests/unit/core/neural_module/test_module_configuration_import.py similarity index 100% rename from tests/unit/core/test_module_configuration_import.py rename to tests/unit/core/neural_module/test_module_configuration_import.py diff --git a/tests/unit/core/test_module_initialization.py b/tests/unit/core/neural_module/test_module_initialization.py similarity index 100% rename from tests/unit/core/test_module_initialization.py rename to tests/unit/core/neural_module/test_module_initialization.py diff --git a/tests/unit/core/test_deprecated.py b/tests/unit/utils/test_deprecated.py similarity index 100% rename from tests/unit/core/test_deprecated.py rename to tests/unit/utils/test_deprecated.py From a7a04e3f974a1101400596d69a2b92f60140634d Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 11:35:53 -0700 Subject: [PATCH 071/106] Fix of NM init_params collection, made it much more robust and faster, fixes of the ObjectRegistry summary, first tests of graph serialization/deserialization Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests2_1.py | 15 ++-- nemo/core/neural_modules.py | 32 ++++++-- nemo/utils/object_registry.py | 10 ++- .../test_neural_graph_serialization.py | 73 +++++++++++++++++++ 4 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 tests/unit/core/neural_graph/test_neural_graph_serialization.py diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index 4e88ddf463e0..b1b94e20599c 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -19,7 +19,7 @@ from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging +from nemo.utils import logging, AppState nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. @@ -47,13 +47,16 @@ print("Serialized:\n", serialized_g1) # Delete everything! -# del g1 -# del dl -# del m1 -# del m2 +del g1 +del dl +del m1 +del m2 + +print(AppState().modules.summary()) +assert len(AppState().modules) == 1 # only the "loss" module # Deserialize graph. -g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=True, name="g1_copy") +g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g1_copy") serialized_g1_copy = g1_copy.serialize() print("Deserialized:\n", serialized_g1_copy) diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index dcb152d84617..56d3da04fc12 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -103,22 +103,44 @@ def __extract_init_params(self): # Get names of arguments of the original module init method. init_keys = getfullargspec(type(self).__init__).args + #(Pdb) localvars + #{'self': , 'batch_size': 1, 'f_name': 'sin', + # 'n': 100, 'x_lo': -4, 'x_hi': 4, 'name': 'tgs1_dl', + #'__class__': } + # Remove self. if "self" in init_keys: init_keys.remove("self") - # Create list of params. - init_params = {}.fromkeys(init_keys) + # Create a list of params and initialize it with a special value. + init_params = {}.fromkeys(init_keys, "__UNSET__") + no_of_unset = len(init_params) # Retrieve values of those params from the call list. + # Do it by removing and analysing the calls from stack one by one. for frame in stack()[1:]: + # Get call "context". localvars = getargvalues(frame[0]).locals - # print("localvars: ", localvars) + # Check if we are in the "context" of the class call. + if "__class__" not in localvars.keys(): + continue + # Check if this is the context of the current "class". + if type(localvars["self"]).__name__ != localvars["__class__"].__name__: + # If own class is not equal to the call context class. + continue + # Ok, got the actual __init__() call!! + # Copy the keys. for key in init_keys: - # Found the variable! - if key in localvars.keys(): + # Found the variable - and it is still unset! + if key in localvars.keys() and init_params[key] == "__UNSET__": # Save the value. init_params[key] = localvars[key] + no_of_unset -= 1 + # That should set all the init_params! + assert no_of_unset == 0 + # Ok, we can terminate. + break # Return parameters. return init_params diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 2d570aba82b1..4898e915e6b5 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -118,7 +118,11 @@ def __eq__(self, other): return super().__eq__(other) def summary(self): - """ Returns a summary of objects on the list. """ + """ + Returns: + A summary of the objects on the list. + """ summary = "Objects:\n" - for obj in self.items(): - summary += " * {} ({})\n".format(obj.name, type(obj).__name) + for obj in self: + summary += " * {} ({})\n".format(obj.name, type(obj).__name__) + return summary \ No newline at end of file diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py new file mode 100644 index 000000000000..54462616ff69 --- /dev/null +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -0,0 +1,73 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest +import time + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode + +@pytest.mark.usefixtures("neural_factory") +class TestNeuralGraphSerialization: + @pytest.mark.unit + def test_graph_serialization_1_simple_graph_no_binding(self): + """ + Tests whether serialization of a simple graph works. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs1_dl") + tn = TaylorNet(dim=4, name="tgs1_m1") + loss = MSELoss(name="tgs1_loss") + + # Create the graph. + with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: + x, t = dl() + prediction1 = tn(x=x) + lss = loss(predictions=prediction1, target=t) + + # Serialize the graph. + serialized_g1 = g1.serialize() + + # Create a second graph - deserialize with reusing. + g2 = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=True, name="g2") + serialized_g2 = g2.serialize() + + # Must be the same. + assert serialized_g1 == serialized_g2 + + # Delete modules. + del dl + del tn + del loss + # Delete graphs as they contain "hard" references to those modules. + del g1 + del g2 + + # Create a third graph - deserialize without reusing, should create new modules. + g3 = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g3") + serialized_g3 = g3.serialize() + + # Must be the same. + assert serialized_g1 == serialized_g3 + + # Deserialize graph - without reusing modules not allowed. + with pytest.raises(KeyError): + _ = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False) + + From a932c8a689238d65be63d31fa78d161585caac22 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 11:44:23 -0700 Subject: [PATCH 072/106] Renamed NG integration tests Signed-off-by: Tomasz Kornuta --- .../{test_neural_graph.py => test_integration_neural_graph.py} | 0 ..._graph_nesting.py => test_integration_neural_graph_nesting.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/core/{test_neural_graph.py => test_integration_neural_graph.py} (100%) rename tests/integration/core/{test_neural_graph_nesting.py => test_integration_neural_graph_nesting.py} (100%) diff --git a/tests/integration/core/test_neural_graph.py b/tests/integration/core/test_integration_neural_graph.py similarity index 100% rename from tests/integration/core/test_neural_graph.py rename to tests/integration/core/test_integration_neural_graph.py diff --git a/tests/integration/core/test_neural_graph_nesting.py b/tests/integration/core/test_integration_neural_graph_nesting.py similarity index 100% rename from tests/integration/core/test_neural_graph_nesting.py rename to tests/integration/core/test_integration_neural_graph_nesting.py From 7c57cdf810f708cb10d2fe4bb55d2cd83975799d Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 11:44:44 -0700 Subject: [PATCH 073/106] formatting Signed-off-by: Tomasz Kornuta --- .../start_here/graph_composition_integration_tests2_1.py | 4 ++-- nemo/core/neural_modules.py | 4 ++-- nemo/utils/object_registry.py | 2 +- .../core/neural_graph/test_neural_graph_serialization.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py index b1b94e20599c..5f163c9a818a 100644 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ b/examples/start_here/graph_composition_integration_tests2_1.py @@ -19,7 +19,7 @@ from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging, AppState +from nemo.utils import AppState, logging nf = NeuralModuleFactory(placement=DeviceType.CPU) # Instantiate the necessary neural modules. @@ -53,7 +53,7 @@ del m2 print(AppState().modules.summary()) -assert len(AppState().modules) == 1 # only the "loss" module +assert len(AppState().modules) == 1 # only the "loss" module # Deserialize graph. g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g1_copy") diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 56d3da04fc12..ffee973912e0 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -103,8 +103,8 @@ def __extract_init_params(self): # Get names of arguments of the original module init method. init_keys = getfullargspec(type(self).__init__).args - #(Pdb) localvars - #{'self': , 'batch_size': 1, 'f_name': 'sin', # 'n': 100, 'x_lo': -4, 'x_hi': 4, 'name': 'tgs1_dl', #'__class__': } diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 4898e915e6b5..2b7f068c3cb6 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -125,4 +125,4 @@ def summary(self): summary = "Objects:\n" for obj in self: summary += " * {} ({})\n".format(obj.name, type(obj).__name__) - return summary \ No newline at end of file + return summary diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 54462616ff69..75a6fe0ad3ac 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -17,12 +17,14 @@ # limitations under the License. # ============================================================================= -import pytest import time +import pytest + from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode + @pytest.mark.usefixtures("neural_factory") class TestNeuralGraphSerialization: @pytest.mark.unit @@ -69,5 +71,3 @@ def test_graph_serialization_1_simple_graph_no_binding(self): # Deserialize graph - without reusing modules not allowed. with pytest.raises(KeyError): _ = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False) - - From 6d785f17080f7f2b87a5882ff14aeae47f2c201d Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 12:23:48 -0700 Subject: [PATCH 074/106] additional tests Signed-off-by: Tomasz Kornuta --- .../test_neural_graph_serialization.py | 119 +++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 75a6fe0ad3ac..0fc8f818df84 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -34,14 +34,14 @@ def test_graph_serialization_1_simple_graph_no_binding(self): """ # Instantiate the necessary neural modules. dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs1_dl") - tn = TaylorNet(dim=4, name="tgs1_m1") + tn = TaylorNet(dim=4, name="tgs1_tn") loss = MSELoss(name="tgs1_loss") # Create the graph. with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: x, t = dl() prediction1 = tn(x=x) - lss = loss(predictions=prediction1, target=t) + _ = loss(predictions=prediction1, target=t) # Serialize the graph. serialized_g1 = g1.serialize() @@ -71,3 +71,118 @@ def test_graph_serialization_1_simple_graph_no_binding(self): # Deserialize graph - without reusing modules not allowed. with pytest.raises(KeyError): _ = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False) + + + @pytest.mark.unit + def test_graph_serialization_2_simple_graph_output_binding(self): + """ + Tests whether serialization of a simple graph with output binding works. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs2_dl") + tn = TaylorNet(dim=4, name="tgs2_tn") + loss = MSELoss(name="tgs2_loss") + + # Create the graph. + with NeuralGraph(operation_mode=OperationMode.inference) as g1: + x, t = dl() + prediction1 = tn(x=x) + _ = loss(predictions=prediction1, target=t) + # Manually bind the selected outputs. + g1.outputs["ix"] = x + g1.outputs["te"] = t + g1.outputs["prediction"] = prediction1 + + # Serialize graph + serialized_g1 = g1.serialize() + + # Create the second graph - deserialize with reusing. + g2 = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=True) + serialized_g2 = g2.serialize() + + # Must be the same. + assert serialized_g1 == serialized_g2 + + @pytest.mark.unit + def test_graph_serialization_3_simple_model_input_output_binding(self): + """ + Tests whether serialization of a simple graph with input and output binding works. + """ + # Instantiate the necessary neural modules. + tn = TaylorNet(dim=4, name="tgs3_tn") + + # Create "model". + with NeuralGraph(operation_mode=OperationMode.both, name="model") as model: + # Manually bind input port: "input" -> "x" + model.inputs["input"] = tn.input_ports["x"] + # Add module to graph and bind it input port 'x'. + y = tn(x=model.inputs["input"]) + # Manual output bind. + model.outputs["output"] = y + + # Serialize the "model". + serialized_model1 = model.serialize() + + # Create the second graph - deserialize with reusing. + model2 = NeuralGraph.deserialize(serialized_model1, reuse_existing_modules=True) + serialized_model2 = model2.serialize() + + # Must be the same. + assert serialized_model1 == serialized_model2 + + @pytest.mark.unit + def test_graph_serialization_4_serialize_graph_after_nesting(self): + """ + Tests whether serialization of a simple graph with input and output binding works. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1) + tn = TaylorNet(dim=4) + loss = MSELoss() + + # Create "model". + with NeuralGraph(operation_mode=OperationMode.both, name="model") as model: + # Manually bind input port: "input" -> "x" + model.inputs["input"] = tn.input_ports["x"] + # Add module to graph and bind it input port 'x'. + y = tn(x=model.inputs["input"]) + # Manual output bind. + # model.outputs["output"] = y # BUG! + + # Serialize "model". + serialized_model = model.serialize() + + # Delete model-related stuff. + del tn + del model + + # Deserialize the "model copy". + model_copy = NeuralGraph.deserialize(serialized_model, name="model_copy") + + # Build the "training graph" - using the model copy. + with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: + # Add modules to graph. + x, t = dl() + # Incorporate modules from the existing "model" graph. + p = model_copy(input=x) + lss = loss(predictions=p, target=t) + + # Serialize the "training graph". + serialized_training = training.serialize() + + # Delete everything. + del dl + del loss + del model_copy + del training + + # Create the second graph - deserialize withoput "module reusing". + training2 = NeuralGraph.deserialize(serialized_training) + serialized_training2 = training2.serialize() + + # import pdb;pdb.set_trace() + # print("1: \n",serialized_training) + # print("2: \n",serialized_training2) + + # Must be the same. + assert serialized_training == serialized_training2 From 5633963be5ece746f64e1fd41f838e2feaed9ace Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 12:24:11 -0700 Subject: [PATCH 075/106] style fix Signed-off-by: Tomasz Kornuta --- .../core/neural_graph/test_neural_graph_serialization.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 0fc8f818df84..760e90704f86 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -72,7 +72,6 @@ def test_graph_serialization_1_simple_graph_no_binding(self): with pytest.raises(KeyError): _ = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False) - @pytest.mark.unit def test_graph_serialization_2_simple_graph_output_binding(self): """ @@ -92,7 +91,7 @@ def test_graph_serialization_2_simple_graph_output_binding(self): g1.outputs["ix"] = x g1.outputs["te"] = t g1.outputs["prediction"] = prediction1 - + # Serialize graph serialized_g1 = g1.serialize() @@ -119,7 +118,7 @@ def test_graph_serialization_3_simple_model_input_output_binding(self): y = tn(x=model.inputs["input"]) # Manual output bind. model.outputs["output"] = y - + # Serialize the "model". serialized_model1 = model.serialize() @@ -157,7 +156,7 @@ def test_graph_serialization_4_serialize_graph_after_nesting(self): del model # Deserialize the "model copy". - model_copy = NeuralGraph.deserialize(serialized_model, name="model_copy") + model_copy = NeuralGraph.deserialize(serialized_model, name="model_copy") # Build the "training graph" - using the model copy. with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: @@ -169,7 +168,7 @@ def test_graph_serialization_4_serialize_graph_after_nesting(self): # Serialize the "training graph". serialized_training = training.serialize() - + # Delete everything. del dl del loss From 8f6ddc540c79608e3f121e3a347dffb866ae66a5 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 12:53:04 -0700 Subject: [PATCH 076/106] A simple test for graph config import and export Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/neural_graph.py | 72 +++++++++-- nemo/core/neural_modules.py | 112 +++++++++--------- .../test_neural_graph_serialization.py | 4 +- 3 files changed, 121 insertions(+), 67 deletions(-) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 6d0895378023..92a06788f0a1 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -21,8 +21,11 @@ ] from collections import OrderedDict, namedtuple +from os import path from typing import Dict, Optional +from ruamel.yaml import YAML + from nemo.core import OperationMode from nemo.core.neural_graph.graph_inputs import GraphInputs from nemo.core.neural_graph.graph_outputs import GraphOutputs @@ -33,6 +36,8 @@ from nemo.utils import logging from nemo.utils.module_port import Connection, ModulePort +YAML = YAML(typ='safe') + class NeuralGraph(NeuralInterface): """ @@ -405,12 +410,19 @@ def export_to_config(self, config_file): Args: config_file: Name (and path) of the config file (YML) to be written to. """ - # Create a dictionary where we will add the whole information. - config = {self.name: {}} - # Get shortcut. - graph = config[self.name] - # Serialize modules. - graph["modules"] = self.__serialize_modules() + # Greate an absolute path. + abs_path_file = path.expanduser(config_file) + + # Serialize the graph. + to_export = self.serialize() + + # All parameters are ok, let's export. + with open(abs_path_file, 'w') as outfile: + YAML.dump(to_export, outfile) + + logging.info( + "Configuration of graph `{}` ({}) exported to {}".format(self.name, type(self).__name__, abs_path_file) + ) def serialize(self): """ Method serializes the whole graph. @@ -524,15 +536,57 @@ def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite """ logging.info("Loading configuration of a new Neural Graph from the `{}` file".format(config_file)) - # TODO: validate the content of the configuration file (its header). - loaded_config = [] # cls.__validate_config_file(config_file, section_name) - # TODO: overwrite params. + # Validate the content of the configuration file (its header). + loaded_config = cls.__validate_config_file(config_file) + # TODO: overwrite params? # "Deserialize" the graph. new_graph = cls.deserialize(loaded_config, reuse_existing_modules, name) + # Return the object. return new_graph + @classmethod + def __validate_config_file(cls, config_file): + """ + Class method validating whether the config file has a proper content (sections, specification etc.). + Raises an ImportError exception when config file is invalid or + incompatible (when called from a particular class). + + Args: + config_file: path (absolute or relative) and name of the config file (YML) + + Returns: + A loaded configuration file (dictionary). + """ + # Greate an absolute path. + abs_path_file = path.expanduser(config_file) + + # Open the config file. + with open(abs_path_file, 'r') as stream: + loaded_config = YAML.load(stream) + + # Check sections. + for section_name in ["header", "modules", "steps", "connections", "inputs", "outputs"]: + if section_name not in loaded_config.keys(): + raise ImportError( + "The loaded config `{}` doesn't contain the required `{}` section".format( + config_file, section_name + ) + ) + + # Parse the "full specification". + spec_list = loaded_config["header"]["full_spec"].split(".") + + # Check if config contains definition of Neural Graph. + if spec_list[-1] != "NeuralGraph": + txt = "The loaded file `{}` contains configuration of ".format(config_file) + txt = txt + "`{}` thus cannot be used for instantiation of Neural Graph".format(spec_list[-1]) + raise ImportError(txt) + + # Success - return the loaded configuration. + return loaded_config + @classmethod def deserialize(cls, configuration, reuse_existing_modules=False, name=None): """ diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index ffee973912e0..9d233ca63eda 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -224,7 +224,7 @@ def export_to_config(self, config_file): YAML.dump(to_export, outfile) logging.info( - "Configuration of module {} ({}) exported to {}".format(self._uuid, type(self).__name__, abs_path_file) + "Configuration of module `{}` ({}) exported to {}".format(self.name, type(self).__name__, abs_path_file) ) def serialize(self): @@ -334,14 +334,70 @@ def import_from_config(cls, config_file, section_name=None, name=None, overwrite Instance of the created NeuralModule object. """ logging.info("Loading configuration of a new Neural Module from the `{}` file".format(config_file)) + # Validate the content of the configuration file (its header). loaded_config = cls.__validate_config_file(config_file, section_name) # "Deserialize" the module. obj = cls.deserialize(loaded_config, name, overwrite_params) + # Return the new module. return obj + @classmethod + def __validate_config_file(cls, config_file, section_name=None): + """ + Class method validating whether the config file has a proper content (sections, specification etc.). + Raises an ImportError exception when config file is invalid or + incompatible (when called from a particular class). + + Args: + config_file: path (absolute or relative) and name of the config file (YML) + + section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) + + Returns: + A loaded configuration file (dictionary). + """ + # Greate an absolute path. + abs_path_file = path.expanduser(config_file) + + # Open the config file. + with open(abs_path_file, 'r') as stream: + loaded_config = YAML.load(stream) + + # Check section. + if section_name is not None: + if section_name not in loaded_config: + raise ImportError( + "The loaded config `{}` doesn't contain the indicated `{}` section".format( + config_file, section_name + ) + ) + # Section exists - use only it for configuration. + loaded_config = loaded_config[section_name] + + # Make sure that the config is valid. + if "header" not in loaded_config: + raise ImportError("The loaded config `{}` doesn't contain the `header` section".format(config_file)) + + if "init_params" not in loaded_config: + raise ImportError("The loaded config `{}` doesn't contain the `init_params` section".format(config_file)) + + # Parse the "full specification". + spec_list = loaded_config["header"]["full_spec"].split(".") + + # Check if config contains data of a compatible class. + if cls.__name__ != "NeuralModule" and spec_list[-1] != cls.__name__: + txt = "The loaded file `{}` contains configuration of ".format(config_file) + txt = txt + "`{}` thus cannot be used for instantiation of an object of type `{}`".format( + spec_list[-1], cls.__name__ + ) + raise ImportError(txt) + + # Success - return configuration. + return loaded_config + @classmethod def deserialize(cls, configuration, name=None, overwrite_params={}): """ @@ -422,60 +478,6 @@ def _deserialize_configuration(cls, serialized_init_params): # In this case configuration = init parameters. return serialized_init_params - @classmethod - def __validate_config_file(cls, config_file, section_name=None): - """ - Class method validating whether the config file has a proper content (sections, specification etc.). - Raises an ImportError exception when config file is invalid or - incompatible (when called from a particular class). - - Args: - config_file: path (absolute or relative) and name of the config file (YML) - - section_name: section in the configuration file storing module configuration (optional, DEFAULT: None) - - Returns: - A loaded configuration file (dictionary). - """ - # Greate an absolute path. - abs_path_file = path.expanduser(config_file) - - # Open the config file. - with open(abs_path_file, 'r') as stream: - loaded_config = YAML.load(stream) - - # Check section. - if section_name is not None: - if section_name not in loaded_config: - raise ImportError( - "The loaded config `{}` doesn't contain the indicated `{}` section".format( - config_file, section_name - ) - ) - # Section exists - use only it for configuration. - loaded_config = loaded_config[section_name] - - # Make sure that the config is valid. - if "header" not in loaded_config: - raise ImportError("The loaded config `{}` doesn't contain the `header` section".format(config_file)) - - if "init_params" not in loaded_config: - raise ImportError("The loaded config `{}` doesn't contain the `init_params` section".format(config_file)) - - # Parse the "full specification". - spec_list = loaded_config["header"]["full_spec"].split(".") - - # Check if config contains data of a compatible class. - if cls.__name__ != "NeuralModule" and spec_list[-1] != cls.__name__: - txt = "The loaded file `{}` contains configuration of ".format(config_file) - txt = txt + "`{}` thus cannot be used for instantiation of an object of type `{}`".format( - spec_list[-1], cls.__name__ - ) - raise ImportError(txt) - - # Success - return configuration. - return loaded_config - @deprecated(version=0.11) @staticmethod def create_ports(**kwargs): diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 760e90704f86..07b1c95e8de1 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -17,12 +17,10 @@ # limitations under the License. # ============================================================================= -import time - import pytest from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode +from nemo.core import NeuralGraph, OperationMode @pytest.mark.usefixtures("neural_factory") From 04985581df6d763ece8fe5d809c22435e6634ff1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2020 17:36:05 -0700 Subject: [PATCH 077/106] add a nmtensor registry Signed-off-by: Jason --- nemo/core/neural_types/neural_type.py | 4 +- nemo/core/neural_types/nmtensor_registry.py | 81 +++++++++++++++++++++ nemo/utils/app_state.py | 14 ++-- 3 files changed, 91 insertions(+), 8 deletions(-) create mode 100755 nemo/core/neural_types/nmtensor_registry.py diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 2848d8f5c4fd..9535d17fc6bc 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -49,9 +49,9 @@ class NeuralType(object): def __str__(self): if self.axes is not None: - return f"axes: {self.axes}; " f" elements_type: {self.elements_type.__class__.__name__}" + return f"axes: {self.axes}; elements_type: {self.elements_type.__class__.__name__}" else: - return f"axes: None; " f" elements_type: {self.elements_type.__class__.__name__}" + return f"axes: None; elements_type: {self.elements_type.__class__.__name__}" def __init__(self, axes: Optional[Tuple] = None, elements_type: ElementType = VoidType(), optional=False): if not isinstance(elements_type, ElementType): diff --git a/nemo/core/neural_types/nmtensor_registry.py b/nemo/core/neural_types/nmtensor_registry.py new file mode 100755 index 000000000000..f9246cf128ec --- /dev/null +++ b/nemo/core/neural_types/nmtensor_registry.py @@ -0,0 +1,81 @@ +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + + +class NmTensorNameRegistry(): + def __init__(self): + """ + Constructor. Initializes the manager. Sets active graph to None. + + TODO: Should probably be a property of a graph + """ + # Create the nmtensor_naming_dict + # which contains a mapping of str to NMTensor.unique_name + self._nmtensor_naming_dict = {"loss": None} # Reserve keyname of 'loss' + self._nmtensor_uniname_set = set() + + # def summary(self): + # """ Prints a nice summary. """ + # desc = "" + # for graph in self: + # desc = desc + "`{}`: {}\n".format(graph.name, graph) + # return desc + + def register(self, tensor: NmTensor): + """TODO + """ + + # Check if object is already in a set. + if tensor.unique_name in self._nmtensor_uniname_set: + pass + + # Finally, add object to the set. + self._nmtensor_uniname_set.add(tensor.unique_name) + + def rename_NmTensor(self, tensor: NmTensor, new_name: str): + """ TODO + """ + # Find old name if exists + old_name = tensor.unique_name + for custom_name, unique_name in self._nmtensor_naming_dict: + if unique_name == tensor.unique_name: + old_name = custom_name + + if old_name != tensor.unique_name: + del self._nmtensor_naming_dict[old_name] + + if new_name in self._nmtensor_naming_dict: + raise KeyError(f"{new_name} already exists in current graph. Please use a unique name") + self._nmtensor_naming_dict["new_name"] = tensor.unique_name + + def __getitem__(self, key): + """ + Object getter function. + + Args: + key: Object name. + + Returns: + Object associated with the key. + """ + # Search for an object with a given name. + if key in self._nmtensor_naming_dict: + key = self._nmtensor_naming_dict[key] + + if key in self._nmtensor_uniname_set: + return key + + raise KeyError("A NmTensor with name `{}` don't exists!".format(key)) diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index db7fc6ff48de..76da9e36065d 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -43,6 +43,8 @@ def __init__(self, device=None): self._module_registry = nemo.utils.ObjectRegistry("module") # Create graph manager (registry with some additional functionality). self._neural_graph_manager = nemo.core.NeuralGraphManager() + # Create NmTensor registry + self._module_registry = nemo.core.neural_types.NmTensorNameRegistry() @property def modules(self): @@ -63,14 +65,14 @@ def graphs(self): return self._neural_graph_manager def register_module(self, module, name): - """ - Registers a module using the provided name. + """ + Registers a module using the provided name. If name is none - generates a new unique name. - + Args: module: A Neural Module object to be registered. name: A "proposition" of module name. - + Returns: A unique name (proposition or newly generated name). """ @@ -80,11 +82,11 @@ def register_graph(self, graph, name): """ Registers a new graph using the provided name. If name is none - generates a new unique name. - + Args: graph: A Neural Graph object to be registered. name: A "proposition" of graph name. - + Returns: A unique name (proposition or newly generated name). """ From 1977db008861a3188f106fb8854791d5c8ff2a53 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 18:17:43 -0700 Subject: [PATCH 078/106] import/export/serialization/deserialization tests cnt, minor fixes here and there Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 12 +- nemo/core/neural_graph/neural_graph.py | 16 +-- .../test_neural_graph_import_export.py | 65 ++++++++++ .../test_neural_graph_serialization.py | 115 ++++++++++++++++-- 4 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 tests/unit/core/neural_graph/test_neural_graph_import_export.py diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 98c2b713ca77..d49d2aef3b99 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -227,10 +227,11 @@ def deserialize(self, serialized_outputs, modules): """ # Check type. if serialized_outputs["type"] == "default": - # We do not need to deserialize. - # self._default_outputs will be recorded automatically during graph execution. - # TODO: check neural types. - return + # We still need to deserialize. + # Use-case: deserialization of a graph with nested graph with bound output. + d = self._default_outputs + else: + d = self._manual_outputs # Iterate through serialized inputs one by one. for i in serialized_outputs["outputs"]: @@ -242,7 +243,6 @@ def deserialize(self, serialized_outputs, modules): n_type = modules[producer_name].output_ports[producer_port_name] # Create a new input. go = GraphOutput(n_type, ModulePort(producer_name, producer_port_name)) - self._manual_outputs[key] = go - # TODO: check neural types. + d[key] = go # Done. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 92a06788f0a1..d3b8539f4ba5 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -31,7 +31,7 @@ from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_modules import NeuralModule -from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor +from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType from nemo.package_info import __version__ as nemo_version from nemo.utils import logging from nemo.utils.module_port import Connection, ModulePort @@ -732,9 +732,9 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # Activate this graph, so all the tensors will be added to this ! self.activate() - # We need to disable the binding of "defeault" ports on per module basis - we will "manually" produce - # them only for ports that are already indicated as the "bound" ones in the inner graph. - # self.default_output_binding = False + # We need to disable the binding of "defeault" ports on per module basis. + # We will "manually" produce (e.g. deserialize) them outside of this function. + self.default_output_binding = False # Now "copy" graph execution order and topology by actually executing each step of the nested graph. for _, module_name in steps.items(): @@ -789,14 +789,14 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): self.deactivate() # Ok, now we can turn automatic binding on. - # self.default_output_binding = True + self.default_output_binding = True - def __str__(self): + def summary(self): """ Prints a nice summary. """ # TODO: a nice summary. ;) desc = "`{}` ({}):\n".format(self.name, len(self._steps)) - for op in self._steps: - desc = desc + " {}\n".format(type(op[0]).__name__) + for num, op in self._steps.items(): + desc = desc + " {}. {}\n".format(num, type(op[0]).__name__) return desc def list_modules(self): diff --git a/tests/unit/core/neural_graph/test_neural_graph_import_export.py b/tests/unit/core/neural_graph/test_neural_graph_import_export.py new file mode 100644 index 000000000000..b41370b99e72 --- /dev/null +++ b/tests/unit/core/neural_graph/test_neural_graph_import_export.py @@ -0,0 +1,65 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# ============================================================================= +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import pytest + +from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet +from nemo.core import NeuralGraph, OperationMode + + +@pytest.mark.usefixtures("neural_factory") +class TestNeuralGraphImportExport: + """ + Class testing Neural Graph configuration import/export. + """ + + @pytest.mark.unit + def test_graph_simple_import_export(self, tmpdir): + """ + Tests whether the Neural Module can instantiate a simple module by loading a configuration file. + + Args: + tmpdir: Fixture which will provide a temporary directory. + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgio1_dl") + tn = TaylorNet(dim=4, name="tgio1_tn") + loss = MSELoss(name="tgio1_loss") + + # Create the graph. + with NeuralGraph(operation_mode=OperationMode.training) as g1: + x, t = dl() + p = tn(x=x) + _ = loss(predictions=p, target=t) + + # Serialize graph + serialized_g1 = g1.serialize() + + # Generate filename in the temporary directory. + tmp_file_name = str(tmpdir.mkdir("export").join("simple_graph.yml")) + + # Export graph to file. + g1.export_to_config(tmp_file_name) + + # Create the second graph - import! + g2 = NeuralGraph.import_from_config(tmp_file_name, reuse_existing_modules=True) + serialized_g2 = g2.serialize() + + # Must be the same. + assert serialized_g1 == serialized_g2 diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 07b1c95e8de1..28e19f8d40af 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -128,40 +128,135 @@ def test_graph_serialization_3_simple_model_input_output_binding(self): assert serialized_model1 == serialized_model2 @pytest.mark.unit - def test_graph_serialization_4_serialize_graph_after_nesting(self): + def test_graph_serialization_4_serialize_graph_after_nesting_with_default_binding_reuse_modules(self): """ - Tests whether serialization of a simple graph with input and output binding works. + Tests whether serialization works in the case when we serialize a graph after a different graph + was nested in it, with additionally bound input and output binding works (default port names). """ # Instantiate the necessary neural modules. - dl = RealFunctionDataLayer(n=100, batch_size=1) - tn = TaylorNet(dim=4) - loss = MSELoss() + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs4_dl") + tn = TaylorNet(dim=4, name="tgs4_tn") + loss = MSELoss(name="tgs4_loss") # Create "model". with NeuralGraph(operation_mode=OperationMode.both, name="model") as model: + # Add module to graph and bind it input port 'x'. + y = tn(x=model) + # NOTE: For some reason after this call both the "tgs4_tn" and "model" objects + # remains on the module/graph registries. + # (So somewhere down there remains a strong reference to module or graph). + # This happens ONLY when passing graph as argument! + # (Check out the next test which actually removes module and graph!). + # Still, that is not an issue, as we do not expect the users + # to delete and recreate modules in their "normal" applications. + + # Build the "training graph" - using the model copy. + with NeuralGraph(operation_mode=OperationMode.training, name="tgs4_training") as training: + # Add modules to graph. + x, t = dl() + # Incorporate modules from the existing "model" graph. + p = model(x=x) + lss = loss(predictions=p, target=t) + + # Serialize the "training graph". + serialized_training = training.serialize() + + # Create the second graph - deserialize withoput "module reusing". + training2 = NeuralGraph.deserialize(serialized_training, reuse_existing_modules=True) + serialized_training2 = training2.serialize() + + # Must be the same. + assert serialized_training == serialized_training2 + + + @pytest.mark.unit + def test_graph_serialization_5_serialize_graph_after_nesting_without_reusing(self): + """ + Tests whether serialization works in the case when we serialize a graph after a different graph + was nested in it, with additionally bound input and output binding works (default port names). + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs5_dl") + tn = TaylorNet(dim=4, name="tgs511_tn") + loss = MSELoss(name="tgs5_loss") + + #from nemo.utils.app_state import AppState + #import pdb; pdb.set_trace() + #print(AppState().modules.summary()) + + # Create "model". + with NeuralGraph(operation_mode=OperationMode.both, name="tgs5_model") as model: + # Manually bind input port: "input" -> "x" + model.inputs["input"] = tn.input_ports["x"] + # Add module to graph and bind it input port 'x'. + y = tn(x=model.inputs["input"]) + # Use the default output name. + + # Build the "training graph" - using the model copy. + with NeuralGraph(operation_mode=OperationMode.training, name="tgs5_training") as training: + # Add modules to graph. + x, t = dl() + # Incorporate modules from the existing "model" graph. + p = model(input=x) + lss = loss(predictions=p, target=t) + + # Serialize the "training graph". + serialized_training = training.serialize() + + # Delete everything. + del dl + del tn + del loss + del model + del training + + # Create the second graph - deserialize withoput "module reusing". + training2 = NeuralGraph.deserialize(serialized_training) + serialized_training2 = training2.serialize() + + # import pdb;pdb.set_trace() + # print("1: \n",serialized_training) + # print("2: \n",serialized_training2) + + # Must be the same. + assert serialized_training == serialized_training2 + + @pytest.mark.unit + def test_graph_serialization_6_serialize_graph_after_nesting_with_manual_binding(self): + """ + Tests whether serialization works in the case when we serialize a graph after a different graph + was nested in it, with additionally bound input and output binding works (manual port names). + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="tgs6_dl") + tn = TaylorNet(dim=4, name="tgs6_tn") + loss = MSELoss(name="tgs6_loss") + + # Create "model". + with NeuralGraph(operation_mode=OperationMode.both, name="tgs6_model") as model: # Manually bind input port: "input" -> "x" model.inputs["input"] = tn.input_ports["x"] # Add module to graph and bind it input port 'x'. y = tn(x=model.inputs["input"]) # Manual output bind. - # model.outputs["output"] = y # BUG! + model.outputs["output"] = y # Serialize "model". serialized_model = model.serialize() # Delete model-related stuff. - del tn del model + del tn # Deserialize the "model copy". - model_copy = NeuralGraph.deserialize(serialized_model, name="model_copy") + model_copy = NeuralGraph.deserialize(serialized_model, name="tgs6_model_copy") # Build the "training graph" - using the model copy. - with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: + with NeuralGraph(operation_mode=OperationMode.training, name="tgs6_training") as training: # Add modules to graph. x, t = dl() # Incorporate modules from the existing "model" graph. - p = model_copy(input=x) + p = model_copy(input=x) # Note: this output should actually be named "output", not "y_pred"! lss = loss(predictions=p, target=t) # Serialize the "training graph". From bd6c12e88c06557b8715d8964fadd098accdab76 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 19:02:37 -0700 Subject: [PATCH 079/106] minor cleanup, unit test: a simple graph with a loop (not passing) Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/neural_graph.py | 26 ++++++++- .../test_neural_graph_serialization.py | 54 +++++++++++++------ 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index d3b8539f4ba5..3014d5a6b0e5 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -368,9 +368,33 @@ def steps(self): @property def tensors(self): - """ Returns the (double) dictionary of all output tensors, aggregated by modules (key) and (output) port name. """ + """ + Property returning a (double) dictionary of all output tensors. + + Returns: + Dictionary of tensors in the format [module_name][output_port_name]. + + """ return self._all_tensors + @property + def tensor_list(self): + """ + Property returning output tensors by extracting them on the fly from the bound outputs. + + Returns: + List of tensors. + + """ + tensor_list = [] + # Get tensors by acessing the producer-ports. + for tensors_per_module in self._all_tensors.values(): + for tensor in tensors_per_module.values(): + # Add it to the list. + tensor_list.append(tensor) + # Return the result. + return tensor_list + @property def operation_mode(self): """ Returns operation mode. """ diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 28e19f8d40af..8bbd6478e608 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -128,7 +128,7 @@ def test_graph_serialization_3_simple_model_input_output_binding(self): assert serialized_model1 == serialized_model2 @pytest.mark.unit - def test_graph_serialization_4_serialize_graph_after_nesting_with_default_binding_reuse_modules(self): + def test_graph_serialization_4_graph_after_nesting_with_default_binding_reuse_modules(self): """ Tests whether serialization works in the case when we serialize a graph after a different graph was nested in it, with additionally bound input and output binding works (default port names). @@ -168,9 +168,8 @@ def test_graph_serialization_4_serialize_graph_after_nesting_with_default_bindin # Must be the same. assert serialized_training == serialized_training2 - @pytest.mark.unit - def test_graph_serialization_5_serialize_graph_after_nesting_without_reusing(self): + def test_graph_serialization_5_graph_after_nesting_without_reusing(self): """ Tests whether serialization works in the case when we serialize a graph after a different graph was nested in it, with additionally bound input and output binding works (default port names). @@ -180,10 +179,6 @@ def test_graph_serialization_5_serialize_graph_after_nesting_without_reusing(sel tn = TaylorNet(dim=4, name="tgs511_tn") loss = MSELoss(name="tgs5_loss") - #from nemo.utils.app_state import AppState - #import pdb; pdb.set_trace() - #print(AppState().modules.summary()) - # Create "model". with NeuralGraph(operation_mode=OperationMode.both, name="tgs5_model") as model: # Manually bind input port: "input" -> "x" @@ -214,15 +209,11 @@ def test_graph_serialization_5_serialize_graph_after_nesting_without_reusing(sel training2 = NeuralGraph.deserialize(serialized_training) serialized_training2 = training2.serialize() - # import pdb;pdb.set_trace() - # print("1: \n",serialized_training) - # print("2: \n",serialized_training2) - # Must be the same. assert serialized_training == serialized_training2 @pytest.mark.unit - def test_graph_serialization_6_serialize_graph_after_nesting_with_manual_binding(self): + def test_graph_serialization_6_graph_after_nesting_with_manual_binding(self): """ Tests whether serialization works in the case when we serialize a graph after a different graph was nested in it, with additionally bound input and output binding works (manual port names). @@ -256,7 +247,7 @@ def test_graph_serialization_6_serialize_graph_after_nesting_with_manual_binding # Add modules to graph. x, t = dl() # Incorporate modules from the existing "model" graph. - p = model_copy(input=x) # Note: this output should actually be named "output", not "y_pred"! + p = model_copy(input=x) # Note: this output should actually be named "output", not "y_pred"! lss = loss(predictions=p, target=t) # Serialize the "training graph". @@ -272,9 +263,38 @@ def test_graph_serialization_6_serialize_graph_after_nesting_with_manual_binding training2 = NeuralGraph.deserialize(serialized_training) serialized_training2 = training2.serialize() - # import pdb;pdb.set_trace() - # print("1: \n",serialized_training) - # print("2: \n",serialized_training2) - # Must be the same. assert serialized_training == serialized_training2 + + @pytest.mark.unit + def test_graph_serialization_7_arbitrary_graph_with_loops(self): + """ + Tests whether serialization works in the case when we serialize a graph after a different graph + was nested in it, with additionally bound input and output binding works (manual port names). + """ + # Instantiate the necessary neural modules. + dl = RealFunctionDataLayer(n=100, batch_size=1, name="dl") + tn = TaylorNet(dim=4, name="tn") + loss = MSELoss(name="loss") + + # Build a graph with a loop. + with NeuralGraph(name="graph") as graph: + # Add modules to graph. + x, t = dl() + # First call to TN. + p1 = tn(x=x) + # Second call to TN. + p2 = tn(x=p1) + # Take output of second, pass it to loss. + lss = loss(predictions=p2, target=t) + + # Make sure all connections are there! + #assert len(graph.tensor_list) == 5 # TODO! NOT TRUE!! + + # Serialize the graph. + #serialized_graph = graph.serialize() + + #import pdb;pdb.set_trace() + #print("1: \n",serialized_graph) + # print("2: \n",serialized_training2) + From 7463dd85ab1f6b75157989f7821ae85163a0ff97 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 19:03:03 -0700 Subject: [PATCH 080/106] style fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/neural_graph.py | 2 +- .../core/neural_graph/test_neural_graph_serialization.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 3014d5a6b0e5..5fee4e5a8084 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -394,7 +394,7 @@ def tensor_list(self): tensor_list.append(tensor) # Return the result. return tensor_list - + @property def operation_mode(self): """ Returns operation mode. """ diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 8bbd6478e608..64052bc88fb8 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -289,12 +289,11 @@ def test_graph_serialization_7_arbitrary_graph_with_loops(self): lss = loss(predictions=p2, target=t) # Make sure all connections are there! - #assert len(graph.tensor_list) == 5 # TODO! NOT TRUE!! + # assert len(graph.tensor_list) == 5 # TODO! NOT TRUE!! # Serialize the graph. - #serialized_graph = graph.serialize() + # serialized_graph = graph.serialize() - #import pdb;pdb.set_trace() - #print("1: \n",serialized_graph) + # import pdb;pdb.set_trace() + # print("1: \n",serialized_graph) # print("2: \n",serialized_training2) - From 17e4dd9823de53751ee056d6b65bc12ac5ee604a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 23:48:51 -0700 Subject: [PATCH 081/106] Refactored the whole solution, switching from module_name:port_name to step_number:module_name:port_name (or in case of tensors: step_number:port_name). In short: enabled neural graphs to handle loops. Polished the code, unit tests and examples working. Added neural tensor type export along each connection between modules Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 62 ++++----- nemo/core/neural_graph/graph_outputs.py | 51 ++++---- nemo/core/neural_graph/neural_graph.py | 123 +++++++++++------- nemo/core/neural_modules.py | 10 +- nemo/core/neural_types/neural_type.py | 55 ++++---- nemo/utils/__init__.py | 2 +- nemo/utils/{module_port.py => connection.py} | 10 +- .../test_graph_outputs_binding.py | 4 +- .../neural_graph/test_neural_graph_nesting.py | 63 ++++----- .../test_neural_graph_serialization.py | 21 ++- 10 files changed, 229 insertions(+), 172 deletions(-) rename nemo/utils/{module_port.py => connection.py} (74%) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 85f2cace3be6..bbd9d8f31939 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -20,10 +20,7 @@ from nemo.core.neural_types import NeuralType from nemo.utils import logging -from nemo.utils.module_port import ModulePort - -# Actually this throws error as module dependency is: core depends on utils :] -# from nemo.core.neural_types import NeuralType +from nemo.utils.connection import StepModulePort class GraphInput(object): @@ -38,21 +35,21 @@ def __init__(self, type): """ # (Neural) Type of input. self._type = type - # List of ModulePort tuples to which this input links to (module name, port name). + # List of StepModulePort tuples to which this input links to (step number, module name, port name). self._consumers = [] - def bind(self, module_ports): - """ Binds the (modules-ports) to this "graph input". + def bind(self, step_module_ports): + """ Binds the (step-module-ports) to this "graph input". Args: - module_ports: A single ModulePort OR a list of ModulePort tuples to be added. + step_module_ports: A single StepModulePort OR a list of StepModulePort tuples to be added. """ # Handle both single port and lists of ports to be bound. - if type(module_ports) is not list: - module_ports = [module_ports] + if type(step_module_ports) is not list: + step_module_ports = [step_module_ports] # Interate through "consumers" on the list and add them to bound input. - for module_port in module_ports: - self._consumers.append(module_port) + for smp in step_module_ports: + self._consumers.append(smp) @property def type(self): @@ -60,8 +57,11 @@ def type(self): return self._type @property - def consumers_ports(self): - """ Returns list of bound modules (i.e. (module name, port name) tupes) """ + def consumers(self): + """ + Returns: + List of bound modules i.e. (step number, module name, port name) tupes. + """ return self._consumers @@ -117,45 +117,47 @@ def definitions(self): # Extract port definitions (Neural Types) from the inputs list. return {k: v.type for k, v in self._inputs.items()} - def has_binding(self, module_name, port_name): + def has_binding(self, step_number: int, port_name): """ - Checks if there is a binding leading to a given module and its given port. + Checks if there is a binding leading to a given step number (module) and its given port. + (module name is redundant, thus skipped in this test). Returns: - key in the list of the (bound) input ports that leads to a given module/port or None if the binding was - not found. + key in the list of the (bound) input ports that leads to a given step (module)/port + or None if the binding was not found. """ for key, binding in self._inputs.items(): - for (module, port) in binding.consumers_ports: - if module == module_name and port == port_name: + for (step, _, port) in binding.consumers: + if step == step_number and port == port_name: return key + # Binding not found. return None def serialize(self): """ Method responsible for serialization of the graph inputs. Returns: - List containing mappings (input -> module.input_port). + List containing mappings (input -> step.module.input_port). """ serialized_inputs = [] # Iterate through "bindings". for key, binding in self._inputs.items(): - for (module, port) in binding.consumers_ports: - # Serialize: input -> module.port. - target = module + "." + port + for (step, module, port) in binding.consumers: + # Serialize: input -> step.module.port. + # TODO: add module name. + target = str(step) + "." + module + "." + port serialized_inputs.append(key + "->" + target) # Return the result. return serialized_inputs @classmethod - def deserialize(cls, serialized_inputs, modules, definitions_only=False): + def deserialize(cls, serialized_inputs, modules): """ Class method responsible for deserialization of graph inputs. Args: serialized_inputs: A list of serialized inputs in the form of ("input->module.input_port") modules: List of modules required for neural type copying/checking. - definitions_only: deserializes/checks only the definitions, without binding the consumers. Returns: Dictionary with deserialized inputs. @@ -165,16 +167,14 @@ def deserialize(cls, serialized_inputs, modules, definitions_only=False): for i in serialized_inputs: # Deserialize! [key, consumer] = i.split("->") - [consumer_name, consumer_port_name] = consumer.split(".") + [consumer_step, consumer_name, consumer_port_name] = consumer.split(".") # Add the input. if key not in inputs.keys(): # Get neural type from module input port definition. n_type = modules[consumer_name].input_ports[consumer_port_name] # Create a new input. inputs[key] = n_type - # Optionally, also bind the "consumers". - if not definitions_only: - # Bound input. - inputs[key].bind(ModulePort(consumer_name, consumer_port_name)) + # Bind the "consumers". + inputs[key].bind(StepModulePort(int(consumer_step), consumer_name, consumer_port_name)) # Done. return inputs diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index d49d2aef3b99..27a39cb9e593 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -19,32 +19,32 @@ from collections.abc import MutableMapping from nemo.utils import logging -from nemo.utils.module_port import ModulePort +from nemo.utils.connection import StepModulePort class GraphOutput(object): """ A helper class represenging a single bound output. """ - def __init__(self, type, producer_port): + def __init__(self, ntype, producer_step_module_port): """ Initializes object. Args: type: a NeuralType object. - producer_port: a producer ModulePort tuple (module name, port name). + producer_step_module_port: a producer StepModulePort tuple (step number (module name), port name). """ - self._type = type - self._producer_port = producer_port + self._ntype = ntype + self._producer_step_module_port = producer_step_module_port @property - def type(self): + def ntype(self): """ Returns NeuralType of that output. """ - return self._type + return self._ntype @property - def producer_port(self): - """ Returns producer port (module name, port name) tuple. """ - return self._producer_port + def producer_step_module_port(self): + """ Returns producer step port (step number (module), port name) tuple. """ + return self._producer_step_module_port class GraphOutputs(MutableMapping): @@ -86,7 +86,7 @@ def __setitem__(self, key, value): if key in self._manual_outputs.keys(): raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) # Ok, set output. - self._manual_outputs[key] = GraphOutput(value.type, value.producer_port) + self._manual_outputs[key] = GraphOutput(value.ntype, value.producer_step_module_port) def __getitem__(self, key): """ Returns GraphOutput - depending whether there are some manual outputs or not. """ @@ -127,15 +127,15 @@ def bind(self, tensors_list, port_names=None): # Check the presence of the port name in "default" dictionary. if name in self._default_outputs.keys(): # Name present - use the name being combination of producer and port names. - name = tensor.producer_name + "_" + tensor.name + name = str(tensor.producer_step_number) + "_" + tensor.producer_name + "_" + tensor.name #last = port name logging.warning( - "Setting unigue name of the default output port `{}` produced by `{}` to `{}`".format( - tensor.name, self._default_outputs[tensor.name]._producer_port.module_name, name + "Setting unigue name of the default output port `{}` produced in step {} by `{}` to `{}`".format( + tensor.name, tensor.producer_step_number, tensor.producer_name, name ) ) # Still, "overwrite" it. - self._default_outputs[name] = GraphOutput(tensor.type, tensor.producer_port) + self._default_outputs[name] = GraphOutput(tensor.ntype, tensor.producer_step_module_port) @property def definitions(self): @@ -144,7 +144,7 @@ def definitions(self): d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs # Extract port definitions (Neural Types). - return {k: v.type for k, v in d.items()} + return {k: v.ntype for k, v in d.items()} @property def tensors(self): @@ -160,10 +160,10 @@ def tensors(self): output_tensors = {} # Get tensors by acessing the producer-ports. for k, v in d.items(): - producer_name = v.producer_port.module_name - producer_port_name = v.producer_port.port_name + producer_step = v.producer_step_module_port.step_number + producer_port_name = v.producer_step_module_port.port_name # Find the right output tensor. - tensor = self._tensors_list[producer_name][producer_port_name] + tensor = self._tensors_list[producer_step][producer_port_name] # Add it to the dictionary. output_tensors[k] = tensor # Return the result. @@ -184,10 +184,10 @@ def tensor_list(self): output_tensor_list = [] # Get tensors by acessing the producer-ports. for k, v in d.items(): - producer_name = v.producer_port.module_name - producer_port_name = v.producer_port.port_name + producer_step = v.producer_step_module_port.step_number + producer_port_name = v.producer_step_module_port.port_name # Find the right output tensor. - tensor = self._tensors_list[producer_name][producer_port_name] + tensor = self._tensors_list[producer_step][producer_port_name] # Add it to the list. output_tensor_list.append(tensor) # Return the result. @@ -212,7 +212,8 @@ def serialize(self): # Iterate through "bindings". for key, binding in d.items(): # Serialize: module.port -> output. - source = binding.producer_port.module_name + "." + binding.producer_port.port_name + smp = binding.producer_step_module_port + source = str(smp.step_number) + "." + smp.module_name + "." + smp.port_name serialized_outputs["outputs"].append(source + "->" + key) # Return the result. return serialized_outputs @@ -237,12 +238,12 @@ def deserialize(self, serialized_outputs, modules): for i in serialized_outputs["outputs"]: # Deserialize! [producer, key] = i.split("->") - [producer_name, producer_port_name] = producer.split(".") + [step_number, producer_name, producer_port_name] = producer.split(".") # Get neural type from module output port definition. n_type = modules[producer_name].output_ports[producer_port_name] # Create a new input. - go = GraphOutput(n_type, ModulePort(producer_name, producer_port_name)) + go = GraphOutput(n_type, StepModulePort(int(step_number), producer_name, producer_port_name)) d[key] = go # Done. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 5fee4e5a8084..7f69e29d62c0 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -34,7 +34,7 @@ from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.module_port import Connection, ModulePort +from nemo.utils.connection import Connection, StepModulePort YAML = YAML(typ='safe') @@ -127,6 +127,9 @@ def nest(self, inner_graph, inner_graph_args): inner_graph: Graph to be copied (will be "nested" in this (self) graph). inner_graph_args: inputs passed to the graph call. """ + # Remember the number of "already present steps". + step_bump = len(self.steps) + # "Copy" the modules from nested graph. for key, module in inner_graph.modules.items(): # Check if module with that name already exists. @@ -154,7 +157,7 @@ def nest(self, inner_graph, inner_graph_args): self.default_output_binding = False # Now "copy" graph execution order and topology by actually executing each step of the nested graph. - for _, module_name in inner_graph.steps.items(): + for step_number, module_name in inner_graph.steps.items(): # Both module and step will be added by the modules' call(). # Get the module. @@ -168,7 +171,7 @@ def nest(self, inner_graph, inner_graph_args): # - checking if we have already tensors leading to that input (in outer graph). for input_port_name in module.input_ports.keys(): # Check if this port was bound in the inner graph. - key = inner_graph.inputs.has_binding(module_name, input_port_name) + key = inner_graph.inputs.has_binding(step_number, input_port_name) # If so, then we must pass whatever was passed to that port in the list of arguments. if key is not None: module_args[input_port_name] = inner_graph_args[key] @@ -179,17 +182,19 @@ def nest(self, inner_graph, inner_graph_args): # Search for producer/port that we should use. for connection in inner_connections: if ( - connection.consumer.module_name == module_name + connection.consumer.step_number == step_number + and connection.consumer.module_name == module_name and connection.consumer.port_name == input_port_name ): # Got the connection! - producer_name = connection.producer.module_name + bumped_step = connection.producer.step_number + step_bump + #producer_name = connection.producer.module_name producer_port_name = connection.producer.port_name break + #import pdb;pdb.set_trace() # Now, the tensor is already produced in outer (i.e. this) graph! - module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + module_args[input_port_name] = self.tensors[bumped_step][producer_port_name] - # import pdb;pdb.set_trace() # Ok, now we have all keyword arguments. We can call() the module. # This will collect all the produced output tensors and add them to this graph. module(**module_args) @@ -205,11 +210,12 @@ def nest(self, inner_graph, inner_graph_args): output_tensors = {} # Iterate through outputs of the inner graph. for key, tensor in inner_graph.output_tensors.items(): - # Find the tensors within this (outer) graph that are outpus by the same producer-port. - producer_name = tensor.producer_name + # Find the tensors within this (outer) graph that are outputs by the same producer-port. + bumped_step = tensor.producer_step_number + step_bump + #producer_name = tensor.producer_name producer_port_name = tensor.name # Get adequate tensor from "outer graph" (self). - output_tensors[key] = self.tensors[producer_name][producer_port_name] + output_tensors[key] = self.tensors[bumped_step][producer_port_name] if len(output_tensors) == 1: # Return a single tensor. @@ -246,21 +252,39 @@ def record_step(self, module): Args: module: Neural modules added to a given graph. + + Returns: + Step number. """ - # Check if module with that name already exists - to avoid potential loops (DAG). - # TODO: Uncomment after we will refactor all the examples, so training/validation graphs won't be added - # to the "default" graph. - # if module.name in self._modules.keys() and self._modules[module.name] is not module: - # raise KeyError("Neural Graph already contains a module named {}".format(module.name)) + # The solution allows loops in the graph. + # This also means that module with that name can already be present in the graph. + if module.name in self._modules.keys(): + # Check if this is the same module. + if self._modules[module.name] is not module: + raise KeyError("Neural Graph already contains a different module with name `{}`!".format(module.name)) - # Add module to list of modules. - self._modules[module.name] = module + else: + # Add module to list of modules. + self._modules[module.name] = module # Add step - store the module name. - self._steps[len(self._steps)] = module.name + step_number = len(self._steps) + self._steps[step_number] = module.name + + # Return the current step number. + return step_number + + @property + def step_number(self): + """ Returns: + Last step number. """ + return len(self._steps) -1 + + def bind_outputs(self, tensors_list): - """ Binds the output tensors. + """ + Binds the output tensors. Args: tensors_list: A single tensor OR a List of tensors to be bound. @@ -268,16 +292,17 @@ def bind_outputs(self, tensors_list): # Handle both single port and lists of ports to be bound. if type(tensors_list) is not list: tensors_list = [tensors_list] - # Add tensors list to of tensors. + + # Add tensors to list of list of tensors. for tensor in tensors_list: # Add tensor to "all" tensors dictionary. - producer_name = tensor.producer_name - if producer_name not in self._all_tensors.keys(): - self._all_tensors[producer_name] = {} + step_number = tensor.producer_step_number + if step_number not in self._all_tensors.keys(): + self._all_tensors[step_number] = {} port_name = tensor.name # Add tensor. - self._all_tensors[producer_name][port_name] = tensor + self._all_tensors[step_number][port_name] = tensor # Bind the tensors as graph outputs. if self.default_output_binding: @@ -533,9 +558,10 @@ def __serialize_connections(self): # "Transform" tensor to the list of connections. for c in tensor.connections(): # Serialize! - source = c.producer.module_name + "." + c.producer.port_name - target = c.consumer.module_name + "." + c.consumer.port_name - serialized_connections.append(source + "->" + target) + source = str(c.producer.step_number) + "." + c.producer.module_name + "." + c.producer.port_name + target = str(c.consumer.step_number) + "." + c.consumer.module_name + "." + c.consumer.port_name + ntype_str = str(tensor.ntype) + serialized_connections.append(source + "->" + target + " | " + ntype_str) return serialized_connections @classmethod @@ -642,7 +668,7 @@ def deserialize(cls, configuration, reuse_existing_modules=False, name=None): steps = new_graph.__deserialize_steps(configuration["steps"]) # Deserialize the connections between modules. - connections = new_graph.__deserialize_connections(configuration["connections"]) + connections = new_graph.__deserialize_connections(configuration["connections"], modules) # Deserialize input bindings - return it in an external entity. inputs = GraphInputs.deserialize(configuration["inputs"], modules) @@ -719,11 +745,12 @@ def __deserialize_steps(self, serialized_steps): # Ok, done. return steps - def __deserialize_connections(self, serialized_connections): + def __deserialize_connections(self, serialized_connections, modules): """ Private method deserializing the connections in the graph. Args: serialized_steps: Dictionary containing serialized connections. + modules: List of modules. Returns: List of connections, in a format enabling graph traversing. @@ -732,13 +759,19 @@ def __deserialize_connections(self, serialized_connections): # Deserialize connections one by one. for c in serialized_connections: # Deserialize! - [producer, consumer] = c.split("->") - [producer_name, producer_port_name] = producer.split(".") - [consumer_name, consumer_port_name] = consumer.split(".") - producer_mp = ModulePort(producer_name, producer_port_name) - consumer_mp = ModulePort(consumer_name, consumer_port_name) + [producer, consumer_type] = c.split("->") + [consumer, ntype_str] = consumer_type.split(" | ") + [producer_step, producer_name, producer_port_name] = producer.split(".") + [consumer_step, consumer_name, consumer_port_name] = consumer.split(".") + producer_mp = StepModulePort(int(producer_step), producer_name, producer_port_name) + consumer_mp = StepModulePort(int(consumer_step), consumer_name, consumer_port_name) + # Get tensor type. + ntype = modules[producer_name].output_ports[producer_port_name] + # Validate if neural type is ok. + assert ntype_str == str(ntype) + # Add connection. - connections.append(Connection(producer_mp, consumer_mp)) + connections.append(Connection(producer_mp, consumer_mp, ntype)) # Ok, done. return connections @@ -747,10 +780,10 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): the provided connections and inputs. Args: - steps: - modules - connections - inputs + steps: List of steps to be executed. + modules: List of modules. + connections: List of connections. + inputs: List of "bound inputs" """ # Activate this graph, so all the tensors will be added to this ! @@ -761,7 +794,7 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): self.default_output_binding = False # Now "copy" graph execution order and topology by actually executing each step of the nested graph. - for _, module_name in steps.items(): + for step, module_name in steps.items(): # Both module and step will be added by the modules' call(). # Get the module. @@ -775,7 +808,7 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # - checking if we have already tensors leading to that input (in outer graph). for input_port_name in module.input_ports.keys(): # Check if this port was bound in the inner graph. - key = inputs.has_binding(module_name, input_port_name) + key = inputs.has_binding(step, input_port_name) # import pdb;pdb.set_trace() # If so, then we must pass the binding! @@ -791,15 +824,17 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # Search for producer/port that we should use. for connection in connections: if ( - connection.consumer.module_name == module_name + connection.consumer.step_number == step + and connection.consumer.module_name == module_name and connection.consumer.port_name == input_port_name ): # Got the connection! - producer_name = connection.producer.module_name + producer_step_number = connection.producer.step_number + # producer_name = connection.producer.module_name producer_port_name = connection.producer.port_name break # Now, the tensor is already produced in outer (i.e. this) graph! - module_args[input_port_name] = self.tensors[producer_name][producer_port_name] + module_args[input_port_name] = self.tensors[producer_step_number][producer_port_name] # End: for # Ok, now we have all keyword arguments. We can call() the module. diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 9d233ca63eda..e1cc8e868f06 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -34,7 +34,7 @@ from nemo.package_info import __version__ as nemo_version from nemo.utils import logging from nemo.utils.decorators.deprecated import deprecated -from nemo.utils.module_port import ModulePort +from nemo.utils.connection import StepModulePort YAML = YAML(typ='safe') @@ -568,7 +568,7 @@ def __call__(self, **kwargs): # The input and output ports definitions can potentially depend on the operation mode! # Record the operation (i.e. add a single module). - self._app_state.active_graph.record_step(self) + step_number = self._app_state.active_graph.record_step(self) ###### PROCESS INPUTS. ###### # Iterate through all passed parameters. @@ -598,7 +598,7 @@ def __call__(self, **kwargs): # Bind the neural graph input port, i.e. remember that a given graph port should pass data # to THIS module-port (when it finally will be connected). - active_graph.inputs[port_name].bind(ModulePort(self.name, port_name)) + active_graph.inputs[port_name].bind(StepModulePort(step_number, self.name, port_name)) # Please note that there are no "consumers" here - this is a "pure binding". @@ -620,7 +620,7 @@ def __call__(self, **kwargs): # Bind the neural graph input port, i.e. remember that a given graph port should pass data # to THIS module-port (when it finally will be connected). - port_content.bind(ModulePort(self.name, port_name)) + port_content.bind(StepModulePort(step_number, self.name, port_name)) # Please note that there are no "consumers" here - this is a "pure binding". @@ -629,7 +629,7 @@ def __call__(self, **kwargs): self.input_ports[port_name].compare_and_raise_error(self.__class__.__name__, port_name, port_content) # Ok, the goal here is to actually "connect": add self (module) as "consumer" to the input tensor. - port_content.add_consumer(self.name, port_name) + port_content.add_consumer(StepModulePort(step_number, self.name, port_name)) else: raise TypeError( "Input '{}' must be of one of three types: NeuralGraph, GraphInput or NmTensor".format(port_name) diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 2848d8f5c4fd..5c4136d7abb4 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -29,7 +29,7 @@ from nemo.core.neural_types.comparison import NeuralTypeComparisonResult from nemo.core.neural_types.elements import * from nemo.utils.app_state import AppState -from nemo.utils.module_port import Connection, ModulePort +from nemo.utils.connection import Connection, StepModulePort class NeuralType(object): @@ -219,8 +219,10 @@ def __init__(self, producer, producer_args, output_port_name, ntype=None): self._producer_args = producer_args self._output_port_name = output_port_name self._uuid = str(uuid.uuid4()) - # List of tuples (consumer name, input port name) - self._consumers_ports = [] + # Remember step at which this tensor was created. + self._step_number = AppState().active_graph.step_number + # List of tuples (step number, module name, input port name) + self._consumers = [] @property def producer(self): @@ -239,34 +241,41 @@ def producer_name(self): return self._producer_name @property - def producer_port(self): + def producer_step_number(self): """ Returns: - A tuple containing producer name and corresponding output port name. + Step number indicating when the tensor was produced. + (It also indicates who produced the tensor.) """ - return ModulePort(self._producer_name, self._output_port_name) + return self._step_number @property - def consumers_ports(self): + def producer_step_module_port(self): """ Returns: - A list of tuples containing consumer name and corresponding input port names. + A tuple containing step number, module name and corresponding output port name. """ - return self._consumers_ports + return StepModulePort(self._step_number, self._producer_name, self._output_port_name) - def add_consumer(self, module_name, input_port_name): + @property + def consumers(self): + """ + Returns: + A list of tuples containing consumer step number, module name and corresponding input port names. """ - Adds tensor "consumer". + return self._consumers - Args: - module_name: Name of the module that accepts the tensor as input. - input_port_name: Name of the module's input port. + def add_consumer(self, step_module_port): + """ + Adds the "consumer" to tensor. + Args: + step_port: Step number, module name and module's input port. """ - self._consumers_ports.append(ModulePort(module_name, input_port_name)) + self._consumers.append(step_module_port) @property - def type(self): + def ntype(self): """ Returns: Neural Type associated with this NmTensor. @@ -275,20 +284,14 @@ def type(self): def connections(self): """ - "Serializes" the tensor to a list of connections (producer/port, consumer/port). + "Serializes" the tensor to a list of connections (step/producer/port, step/consumer/port). + """ connections = [] - for cp in self._consumers_ports: - connections.append(Connection(self.producer_port, cp)) + for con_mod_port in self._consumers: + connections.append(Connection(self.producer_step_module_port, con_mod_port, self.ntype)) return connections - # @classmethod - # def deserialize(cls): - # """ - # Deserializes tensor from a dictionary (yaml structure). - # """ - # return 2 - @property def producer_args(self): """ diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index b5cca20046ac..663d2a7e4cc0 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -25,4 +25,4 @@ from .helpers import * from nemo.utils.app_state import AppState from nemo.utils.object_registry import ObjectRegistry -from nemo.utils.module_port import ModulePort +from nemo.utils.connection import * diff --git a/nemo/utils/module_port.py b/nemo/utils/connection.py similarity index 74% rename from nemo/utils/module_port.py rename to nemo/utils/connection.py index 9f5a7323d00a..6db23aca00a2 100644 --- a/nemo/utils/module_port.py +++ b/nemo/utils/connection.py @@ -17,16 +17,18 @@ # ============================================================================= __all__ = [ - 'ModulePort', + 'StepModulePort', + 'Connection', ] from collections import namedtuple -# Tuple used for storing "module name" and its "port name". +# Tuple used for storing "step number", "module name" and "port name". # (used in NmTensor's producer/consumer, port binding etc.). -ModulePort = namedtuple('ModulePort', ["module_name", "port_name"]) +# Module name is redundant, as it can be recovered from the step number. +StepModulePort = namedtuple('StepModulePort', ["step_number", "module_name", "port_name"]) # Tuple used for connection between a single producer and a single consummer consumer. # (used in NmTensor's producer/consumer, port binding etc.). -Connection = namedtuple('Connection', ["producer", "consumer"]) +Connection = namedtuple('Connection', ["producer", "consumer", "ntype"]) diff --git a/tests/unit/core/neural_graph/test_graph_outputs_binding.py b/tests/unit/core/neural_graph/test_graph_outputs_binding.py index 7ee2678a6de9..b9e00513f439 100644 --- a/tests/unit/core/neural_graph/test_graph_outputs_binding.py +++ b/tests/unit/core/neural_graph/test_graph_outputs_binding.py @@ -51,6 +51,8 @@ def test_graph_outputs1_binding(self): del bound_outputs["loss"] assert len(bound_outputs) == 4 + assert len(bound_outputs.tensors) == 4 + assert len(bound_outputs.tensor_list) == 4 defs = bound_outputs.definitions assert defs["x"].compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME @@ -100,7 +102,7 @@ def test_graph_outputs2_binding(self): (loss, "loss", lss), ]: # Compare definitions - from outputs. - assert g1.outputs[port].type.compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME + assert g1.outputs[port].ntype.compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME # Compare definitions - from output_ports. assert g1.output_ports[port].compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME # Compare definitions - from output_tensors. diff --git a/tests/unit/core/neural_graph/test_neural_graph_nesting.py b/tests/unit/core/neural_graph/test_neural_graph_nesting.py index 07a501027aaf..cacc5e911263 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_nesting.py +++ b/tests/unit/core/neural_graph/test_neural_graph_nesting.py @@ -138,10 +138,10 @@ def test_graph_nesting3_topology_copy_one_module_default_outputs(self): assert len(g1.outputs) == len(g2.outputs) for port in ["x", "y"]: # Definitions are the same: test two "paths" of accessing the type. - assert g1.outputs[port].type.compare(g1.output_ports[port]) == NeuralTypeComparisonResult.SAME + assert g1.outputs[port].ntype.compare(g1.output_ports[port]) == NeuralTypeComparisonResult.SAME assert g1.output_ports[port].compare(g2.output_ports[port]) == NeuralTypeComparisonResult.SAME - assert g1.outputs[port].type.compare(g2.outputs[port].type) == NeuralTypeComparisonResult.SAME + assert g1.outputs[port].ntype.compare(g2.outputs[port].ntype) == NeuralTypeComparisonResult.SAME # At the same time - those have to be two different port objects! assert g1.outputs[port] is not g2.outputs[port] # And different tensors (as those are "internally produced tensors"!) @@ -181,7 +181,7 @@ def test_graph_nesting4_topology_copy_one_module_manual_outputs(self): for inter_port, outer_port in [("inner_x", "outer_x")]: # Definitions are the same: test two "paths" of accessing the type. assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME - assert g1.outputs[inter_port].type.compare(g2.outputs[outer_port].type) == NeuralTypeComparisonResult.SAME + assert g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME # At the same time - those have to be two different port objects! assert g1.outputs[inter_port] is not g2.outputs[outer_port] # And different tensors (as those are "internally produced tensors"!) @@ -220,7 +220,7 @@ def test_graph_nesting4_1_topology_copy_one_module_manual_outputs_bound_only_in_ for inter_port, outer_port in [("inner_x", "inner_x"), ("inner_t", "inner_t")]: # Definitions are the same: test two "paths" of accessing the type. assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME - assert g1.outputs[inter_port].type.compare(g2.outputs[outer_port].type) == NeuralTypeComparisonResult.SAME + assert g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME # At the same time - those have to be two different port objects! assert g1.outputs[inter_port] is not g2.outputs[outer_port] # And different tensors (as those are "internally produced tensors"!) @@ -254,13 +254,15 @@ def test_graph_nesting5_topology_copy_one_module_default_inputs(self): assert len(g1.inputs) == len(g2.inputs) assert g1.input_ports["x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME assert g2.input_ports["x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME - # At the same time - those point to the same module-port. - assert g1.inputs.has_binding(tn.name, "x") - assert g2.inputs.has_binding(tn.name, "x") - assert g1.inputs["x"].consumers_ports[0].module_name == tn.name - assert g1.inputs["x"].consumers_ports[0].port_name == "x" - assert g2.inputs["x"].consumers_ports[0].module_name == tn.name - assert g2.inputs["x"].consumers_ports[0].port_name == "x" + # At the same time - those point to the same step-module-port. + assert g1.inputs.has_binding(0, "x") + assert g2.inputs.has_binding(0, "x") + assert g1.inputs["x"].consumers[0].step_number == 0 + assert g1.inputs["x"].consumers[0].module_name == tn.name + assert g1.inputs["x"].consumers[0].port_name == "x" + assert g2.inputs["x"].consumers[0].step_number == 0 + assert g2.inputs["x"].consumers[0].module_name == tn.name + assert g2.inputs["x"].consumers[0].port_name == "x" # Make sure that outputs are ok. assert len(g1.outputs) == len(g2.outputs) @@ -308,12 +310,14 @@ def test_graph_nesting6_topology_copy_one_module_manual_inputs(self): assert g1.input_ports["inner_x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME assert g2.input_ports["outer_x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME # At the same time - those point to the same module-port. - assert g1.inputs.has_binding(tn.name, "x") - assert g2.inputs.has_binding(tn.name, "x") - assert g1.inputs["inner_x"].consumers_ports[0].module_name == tn.name - assert g1.inputs["inner_x"].consumers_ports[0].port_name == "x" - assert g2.inputs["outer_x"].consumers_ports[0].module_name == tn.name - assert g2.inputs["outer_x"].consumers_ports[0].port_name == "x" + assert g1.inputs.has_binding(0, "x") + assert g2.inputs.has_binding(0, "x") + assert g1.inputs["inner_x"].consumers[0].step_number == 0 + assert g1.inputs["inner_x"].consumers[0].module_name == tn.name + assert g1.inputs["inner_x"].consumers[0].port_name == "x" + assert g2.inputs["outer_x"].consumers[0].step_number == 0 + assert g2.inputs["outer_x"].consumers[0].module_name == tn.name + assert g2.inputs["outer_x"].consumers[0].port_name == "x" # Make sure that outputs are ok. assert len(g1.outputs) == len(g2.outputs) @@ -380,11 +384,11 @@ def test_graph_nesting7_topology_copy_one_module_all_manual_connect(self): # Check the "internal tensors". assert y_pred2 is not y_pred1 - assert g2.tensors["tgn7_ds"]["x"] == x - assert g2.tensors["tgn7_ds"]["y"] == y - assert g2.tensors["tgn7_loss"]["loss"] == lss + assert g2.tensors[0]["x"] == x + assert g2.tensors[0]["y"] == y + assert g2.tensors[2]["loss"] == lss # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! - assert g2.tensors["tgn7_tn"]["y_pred"] == y_pred2 + assert g2.tensors[1]["y_pred"] == y_pred2 # Update g2: manually bound only one output. with g2: @@ -422,6 +426,7 @@ def test_graph_nesting8_topology_copy_two_modules(self): # Create the "outer graph". with NeuralGraph(operation_mode=OperationMode.training, name="tgn8_g2") as g2: x, y = ds() + # Nest the inner graph. y_pred2, lss2 = g1(inner_x=x, inner_target=y) # Manually bind the output ports. g2.outputs["outer_y_pred"] = y_pred2 @@ -439,12 +444,12 @@ def test_graph_nesting8_topology_copy_two_modules(self): # Check the "internal tensors". assert y_pred2 is not y_pred1 assert lss2 is not lss1 - assert g2.tensors["tgn8_ds"]["x"] == x - assert g2.tensors["tgn8_ds"]["y"] == y + assert g2.tensors[0]["x"] == x + assert g2.tensors[0]["y"] == y # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! - assert g2.tensors["tgn8_tn"]["y_pred"] == y_pred2 + assert g2.tensors[1]["y_pred"] == y_pred2 # Analogically with "loss". - assert g2.tensors["tgn8_loss"]["loss"] == lss2 + assert g2.tensors[2]["loss"] == lss2 @pytest.mark.unit def test_graph_nesting9_topology_copy_whole_graph(self): @@ -486,13 +491,13 @@ def test_graph_nesting9_topology_copy_whole_graph(self): # Check the "internal tensors". assert y_pred2 is not y_pred1 assert lss2 is not lss1 - assert g2.tensors["tgn9_ds"]["x"].type.compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME - assert g2.tensors["tgn9_ds"]["y"].type.compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert g2.tensors[0]["x"].ntype.compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g2.tensors[0]["y"].ntype.compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! assert ( - g2.tensors["tgn9_tn"]["y_pred"].type.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + g2.tensors[1]["y_pred"].ntype.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME ) # Analogically with "loss". assert ( - g2.tensors["tgn9_loss"]["loss"].type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + g2.tensors[2]["loss"].ntype.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME ) diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 64052bc88fb8..24bbe0a4a510 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -259,7 +259,7 @@ def test_graph_serialization_6_graph_after_nesting_with_manual_binding(self): del model_copy del training - # Create the second graph - deserialize withoput "module reusing". + # Create the second graph - deserialize without "module reusing". training2 = NeuralGraph.deserialize(serialized_training) serialized_training2 = training2.serialize() @@ -289,11 +289,20 @@ def test_graph_serialization_7_arbitrary_graph_with_loops(self): lss = loss(predictions=p2, target=t) # Make sure all connections are there! - # assert len(graph.tensor_list) == 5 # TODO! NOT TRUE!! + assert len(graph.tensor_list) == 5 + # 4 would mean that we have overwritten the "p1" (tn->y_pred) tensor! # Serialize the graph. - # serialized_graph = graph.serialize() + serialized_graph = graph.serialize() + + # Create the second graph - deserialize with "module reusing". + graph2 = NeuralGraph.deserialize(serialized_graph, reuse_existing_modules=True) + serialized_graph2 = graph2.serialize() + + # Must be the same. + assert serialized_graph == serialized_graph2 + + #import pdb;pdb.set_trace() + #print("1: \n",serialized_graph) + # print("2: \n",serialized_graph2) - # import pdb;pdb.set_trace() - # print("1: \n",serialized_graph) - # print("2: \n",serialized_training2) From 7b94c5783191d186803ee6da59f7bc81657c88da Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 28 Apr 2020 23:49:10 -0700 Subject: [PATCH 082/106] format fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 4 +++- nemo/core/neural_graph/neural_graph.py | 10 ++++------ nemo/core/neural_modules.py | 2 +- nemo/utils/connection.py | 2 +- .../neural_graph/test_neural_graph_nesting.py | 16 ++++++++-------- .../test_neural_graph_serialization.py | 7 +++---- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 27a39cb9e593..ebf105d84303 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -127,7 +127,9 @@ def bind(self, tensors_list, port_names=None): # Check the presence of the port name in "default" dictionary. if name in self._default_outputs.keys(): # Name present - use the name being combination of producer and port names. - name = str(tensor.producer_step_number) + "_" + tensor.producer_name + "_" + tensor.name #last = port name + name = ( + str(tensor.producer_step_number) + "_" + tensor.producer_name + "_" + tensor.name + ) # last = port name logging.warning( "Setting unigue name of the default output port `{}` produced in step {} by `{}` to `{}`".format( diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 7f69e29d62c0..bd45b769e24e 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -188,10 +188,10 @@ def nest(self, inner_graph, inner_graph_args): ): # Got the connection! bumped_step = connection.producer.step_number + step_bump - #producer_name = connection.producer.module_name + # producer_name = connection.producer.module_name producer_port_name = connection.producer.port_name break - #import pdb;pdb.set_trace() + # import pdb;pdb.set_trace() # Now, the tensor is already produced in outer (i.e. this) graph! module_args[input_port_name] = self.tensors[bumped_step][producer_port_name] @@ -212,7 +212,7 @@ def nest(self, inner_graph, inner_graph_args): for key, tensor in inner_graph.output_tensors.items(): # Find the tensors within this (outer) graph that are outputs by the same producer-port. bumped_step = tensor.producer_step_number + step_bump - #producer_name = tensor.producer_name + # producer_name = tensor.producer_name producer_port_name = tensor.name # Get adequate tensor from "outer graph" (self). output_tensors[key] = self.tensors[bumped_step][producer_port_name] @@ -278,9 +278,7 @@ def record_step(self, module): def step_number(self): """ Returns: Last step number. """ - return len(self._steps) -1 - - + return len(self._steps) - 1 def bind_outputs(self, tensors_list): """ diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index e1cc8e868f06..4a577fd4704c 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -33,8 +33,8 @@ from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.decorators.deprecated import deprecated from nemo.utils.connection import StepModulePort +from nemo.utils.decorators.deprecated import deprecated YAML = YAML(typ='safe') diff --git a/nemo/utils/connection.py b/nemo/utils/connection.py index 6db23aca00a2..ada4e0c0dbb7 100644 --- a/nemo/utils/connection.py +++ b/nemo/utils/connection.py @@ -25,7 +25,7 @@ # Tuple used for storing "step number", "module name" and "port name". # (used in NmTensor's producer/consumer, port binding etc.). -# Module name is redundant, as it can be recovered from the step number. +# Module name is redundant, as it can be recovered from the step number. StepModulePort = namedtuple('StepModulePort', ["step_number", "module_name", "port_name"]) diff --git a/tests/unit/core/neural_graph/test_neural_graph_nesting.py b/tests/unit/core/neural_graph/test_neural_graph_nesting.py index cacc5e911263..c5d3d4b6e2bd 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_nesting.py +++ b/tests/unit/core/neural_graph/test_neural_graph_nesting.py @@ -181,7 +181,9 @@ def test_graph_nesting4_topology_copy_one_module_manual_outputs(self): for inter_port, outer_port in [("inner_x", "outer_x")]: # Definitions are the same: test two "paths" of accessing the type. assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME - assert g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME + assert ( + g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME + ) # At the same time - those have to be two different port objects! assert g1.outputs[inter_port] is not g2.outputs[outer_port] # And different tensors (as those are "internally produced tensors"!) @@ -220,7 +222,9 @@ def test_graph_nesting4_1_topology_copy_one_module_manual_outputs_bound_only_in_ for inter_port, outer_port in [("inner_x", "inner_x"), ("inner_t", "inner_t")]: # Definitions are the same: test two "paths" of accessing the type. assert g1.output_ports[inter_port].compare(g2.output_ports[outer_port]) == NeuralTypeComparisonResult.SAME - assert g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME + assert ( + g1.outputs[inter_port].ntype.compare(g2.outputs[outer_port].ntype) == NeuralTypeComparisonResult.SAME + ) # At the same time - those have to be two different port objects! assert g1.outputs[inter_port] is not g2.outputs[outer_port] # And different tensors (as those are "internally produced tensors"!) @@ -494,10 +498,6 @@ def test_graph_nesting9_topology_copy_whole_graph(self): assert g2.tensors[0]["x"].ntype.compare(ds.output_ports["x"]) == NeuralTypeComparisonResult.SAME assert g2.tensors[0]["y"].ntype.compare(ds.output_ports["y"]) == NeuralTypeComparisonResult.SAME # Internally the name "y_pred" is used, not the "bound output name": "inner_y_pred"! - assert ( - g2.tensors[1]["y_pred"].ntype.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - ) + assert g2.tensors[1]["y_pred"].ntype.compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME # Analogically with "loss". - assert ( - g2.tensors[2]["loss"].ntype.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME - ) + assert g2.tensors[2]["loss"].ntype.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 24bbe0a4a510..76852b9225a8 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -289,7 +289,7 @@ def test_graph_serialization_7_arbitrary_graph_with_loops(self): lss = loss(predictions=p2, target=t) # Make sure all connections are there! - assert len(graph.tensor_list) == 5 + assert len(graph.tensor_list) == 5 # 4 would mean that we have overwritten the "p1" (tn->y_pred) tensor! # Serialize the graph. @@ -302,7 +302,6 @@ def test_graph_serialization_7_arbitrary_graph_with_loops(self): # Must be the same. assert serialized_graph == serialized_graph2 - #import pdb;pdb.set_trace() - #print("1: \n",serialized_graph) + # import pdb;pdb.set_trace() + # print("1: \n",serialized_graph) # print("2: \n",serialized_graph2) - From e5917fe37f09ef61c796613801d4b08b19ea7f67 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 29 Apr 2020 00:34:52 -0700 Subject: [PATCH 083/106] Added neural type export to config file for both graph bound input and output ports, Polishes, cleanups, unit tests working Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 38 ++++++++++------- nemo/core/neural_graph/graph_outputs.py | 54 +++++++++++++++---------- nemo/core/neural_modules.py | 2 +- tests/unit/core/test_nm_tensor.py | 20 ++++----- 4 files changed, 67 insertions(+), 47 deletions(-) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index bbd9d8f31939..f0540ec6b29a 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -26,7 +26,7 @@ class GraphInput(object): """ A helper class represenging a single bound input. """ - def __init__(self, type): + def __init__(self, ntype): """ Initializes object. @@ -34,7 +34,7 @@ def __init__(self, type): type: a NeuralType object. """ # (Neural) Type of input. - self._type = type + self._ntype = ntype # List of StepModulePort tuples to which this input links to (step number, module name, port name). self._consumers = [] @@ -52,9 +52,9 @@ def bind(self, step_module_ports): self._consumers.append(smp) @property - def type(self): + def ntype(self): """ Returns NeuralType of that input. """ - return self._type + return self._ntype @property def consumers(self): @@ -71,7 +71,9 @@ class GraphInputs(MutableMapping): ''' def __init__(self): - """ Initializes the mapping. """ + """ + Initializes an empty dictionary. + """ self._inputs = {} def __setitem__(self, key, value): @@ -89,12 +91,12 @@ def __setitem__(self, key, value): if isinstance(value, NeuralType): val_type = value elif isinstance(value, GraphInput): - val_type = value.type + val_type = value.ntype else: raise TypeError("Port `{}` definition must be must be a NeuralType or GraphInput type".format(key)) # Ok, add definition to list of mapped (module, port)s. # Note: for now, there are no mapped modules, so copy only (neural) type. - self._inputs[key] = GraphInput(type=val_type) + self._inputs[key] = GraphInput(ntype=val_type) def __getitem__(self, key): """ Returns bound input. """ @@ -115,7 +117,7 @@ def __len__(self): def definitions(self): """ Property returns definitions of the input ports by extracting them on the fly from list. """ # Extract port definitions (Neural Types) from the inputs list. - return {k: v.type for k, v in self._inputs.items()} + return {k: v.ntype for k, v in self._inputs.items()} def has_binding(self, step_number: int, port_name): """ @@ -140,13 +142,15 @@ def serialize(self): List containing mappings (input -> step.module.input_port). """ serialized_inputs = [] - # Iterate through "bindings". + # Iterate through "bindings" (GraphInputs). for key, binding in self._inputs.items(): + # Get type. + ntype_str = str(binding.ntype) for (step, module, port) in binding.consumers: - # Serialize: input -> step.module.port. - # TODO: add module name. + # Serialize: input -> step.module.port | ntype target = str(step) + "." + module + "." + port - serialized_inputs.append(key + "->" + target) + # Serialize! + serialized_inputs.append(key + "->" + target + " | " + ntype_str) # Return the result. return serialized_inputs @@ -166,14 +170,18 @@ def deserialize(cls, serialized_inputs, modules): # Iterate through serialized inputs one by one. for i in serialized_inputs: # Deserialize! - [key, consumer] = i.split("->") + [key, consumer_ntype] = i.split("->") + [consumer, ntype_str] = consumer_ntype.split(" | ") [consumer_step, consumer_name, consumer_port_name] = consumer.split(".") # Add the input. if key not in inputs.keys(): # Get neural type from module input port definition. - n_type = modules[consumer_name].input_ports[consumer_port_name] + ntype = modules[consumer_name].input_ports[consumer_port_name] + # Make sure the graph bound port type matches the deserialized type. + assert ntype_str == str(ntype) + # Create a new input. - inputs[key] = n_type + inputs[key] = ntype # Bind the "consumers". inputs[key].bind(StepModulePort(int(consumer_step), consumer_name, consumer_port_name)) # Done. diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index ebf105d84303..1184aed1b5d9 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -58,11 +58,16 @@ class GraphOutputs(MutableMapping): will return/work on "default" outputs. ''' - def __init__(self, tensors_list): - """ Initializes two (empty) dictionaries. """ + def __init__(self, tensors_ref): + """ + Initializes two (empty) dictionaries. + + Args: + tensors_ref - reference to neural graph's tensor (dict of dict). + """ - # List of tensors - passed from the external neural graph object. - self._tensors_list = tensors_list + # Tensors[step][output_port_name] passed from the external neural graph object. + self._tensors_ref = tensors_ref # This dictionary stores the output tensors collected during the "default" tensor recording. # As they are using the default port names, the second/next tensor published on the same port @@ -112,18 +117,18 @@ def __len__(self): else: # Use default dict. return len(self._default_outputs) - def bind(self, tensors_list, port_names=None): + def bind(self, tensors_ref, port_names=None): """ Binds the default outputs. Args: - tensors_list: List of tensors to be added. + tensors_ref: List of tensors to be added. port_names: List of port names (visible outside). If None: using internal tensor "output port names". """ # Set names. if port_names is None: - port_names = [tensor.name for tensor in tensors_list] + port_names = [tensor.name for tensor in tensors_ref] - for name, tensor in zip(port_names, tensors_list): + for name, tensor in zip(port_names, tensors_ref): # Check the presence of the port name in "default" dictionary. if name in self._default_outputs.keys(): # Name present - use the name being combination of producer and port names. @@ -165,7 +170,7 @@ def tensors(self): producer_step = v.producer_step_module_port.step_number producer_port_name = v.producer_step_module_port.port_name # Find the right output tensor. - tensor = self._tensors_list[producer_step][producer_port_name] + tensor = self._tensors_ref[producer_step][producer_port_name] # Add it to the dictionary. output_tensors[k] = tensor # Return the result. @@ -189,7 +194,7 @@ def tensor_list(self): producer_step = v.producer_step_module_port.step_number producer_port_name = v.producer_step_module_port.port_name # Find the right output tensor. - tensor = self._tensors_list[producer_step][producer_port_name] + tensor = self._tensors_ref[producer_step][producer_port_name] # Add it to the list. output_tensor_list.append(tensor) # Return the result. @@ -199,9 +204,9 @@ def serialize(self): """ Method responsible for serialization of the graph outputs. Returns: - List containing mappings (module.output_port -> output). + List containing mappings (step.module.output_port -> output | ntype). """ - serialized_outputs = {"outputs": []} + serialized_outputs = {"mappings": []} # Get the right output dictionary. if len(self._manual_outputs) > 0: @@ -211,12 +216,15 @@ def serialize(self): serialized_outputs["type"] = "default" d = self._default_outputs - # Iterate through "bindings". + # Iterate through "bindings" (GraphOutputs). for key, binding in d.items(): - # Serialize: module.port -> output. + # Serialize: step.module.port -> output | ntype. smp = binding.producer_step_module_port source = str(smp.step_number) + "." + smp.module_name + "." + smp.port_name - serialized_outputs["outputs"].append(source + "->" + key) + # Get type. + ntype_str = str(binding.ntype) + # Serialize! + serialized_outputs["mappings"].append(source + "->" + key + " | " + ntype_str) # Return the result. return serialized_outputs @@ -225,7 +233,7 @@ def deserialize(self, serialized_outputs, modules): Method responsible for deserialization of graph outputs. Args: - serialized_outputs: A list of serialized outputs in the form of ("module.output_port->key") + serialized_outputs: A list of serialized outputs in the form of ("step.module.output_port->key | ntype") modules: List of modules required for neural type copying/checking. """ # Check type. @@ -237,15 +245,19 @@ def deserialize(self, serialized_outputs, modules): d = self._manual_outputs # Iterate through serialized inputs one by one. - for i in serialized_outputs["outputs"]: + for i in serialized_outputs["mappings"]: # Deserialize! - [producer, key] = i.split("->") + [producer, key_ntype] = i.split("->") + [key, ntype_str] = key_ntype.split(" | ") [step_number, producer_name, producer_port_name] = producer.split(".") - # Get neural type from module output port definition. - n_type = modules[producer_name].output_ports[producer_port_name] + ntype = modules[producer_name].output_ports[producer_port_name] + + # Make sure the graph bound port type matches the deserialized type. + assert ntype_str == str(ntype) + # Create a new input. - go = GraphOutput(n_type, StepModulePort(int(step_number), producer_name, producer_port_name)) + go = GraphOutput(ntype, StepModulePort(int(step_number), producer_name, producer_port_name)) d[key] = go # Done. diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 4a577fd4704c..2bd71d9a6906 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -615,7 +615,7 @@ def __call__(self, **kwargs): # Compare input port definition with the received definition. self.input_ports[port_name].compare_and_raise_error( - self.__class__.__name__, port_name, port_content.type + self.__class__.__name__, port_name, port_content.ntype ) # Bind the neural graph input port, i.e. remember that a given graph port should pass data diff --git a/tests/unit/core/test_nm_tensor.py b/tests/unit/core/test_nm_tensor.py index 018dd3dd555c..647905fa51c2 100644 --- a/tests/unit/core/test_nm_tensor.py +++ b/tests/unit/core/test_nm_tensor.py @@ -81,8 +81,8 @@ def test_nm_tensors_producer_consumers(self): lss2 = loss2(predictions=y_pred, target=y) # Check tensor x producer and consumers. - p = x.producer_port - cs = x.consumers_ports + p = x.producer_step_module_port + cs = x.consumers assert p.module_name == "source" assert p.port_name == "x" assert len(cs) == 1 @@ -90,8 +90,8 @@ def test_nm_tensors_producer_consumers(self): assert cs[0].port_name == "x" # Check tensor y producer and consumers. - p = y.producer_port - cs = y.consumers_ports + p = y.producer_step_module_port + cs = y.consumers assert p.module_name == "source" assert p.port_name == "y" assert len(cs) == 2 @@ -101,8 +101,8 @@ def test_nm_tensors_producer_consumers(self): assert cs[1].port_name == "target" # Check tensor y_pred producer and consumers. - p = y_pred.producer_port - cs = y_pred.consumers_ports + p = y_pred.producer_step_module_port + cs = y_pred.consumers assert p.module_name == "tm" assert p.port_name == "y_pred" assert len(cs) == 2 @@ -127,7 +127,7 @@ def test_nm_tensors_types(self): lss = loss(predictions=y_pred, target=y) # Check types. - assert x.type.compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME - assert y.type.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME - assert y_pred.type.compare(trainable_module.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME - assert lss.type.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + assert x.ntype.compare(data_source.output_ports["x"]) == NeuralTypeComparisonResult.SAME + assert y.ntype.compare(data_source.output_ports["y"]) == NeuralTypeComparisonResult.SAME + assert y_pred.ntype.compare(trainable_module.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME + assert lss.ntype.compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME From 35b31c9d79cb9ccfce322eb97c95c560babddce7 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Wed, 29 Apr 2020 01:03:49 -0700 Subject: [PATCH 084/106] jasper polished Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 28be2eec9220..99e7a2fe5097 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -62,49 +62,40 @@ ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) greedy_decoder = nemo_asr.GreedyCTCDecoder() -# Create the Jasper composite module. +# Create the Jasper "model". with NeuralGraph(operation_mode=OperationMode.both) as Jasper: - i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper, length=Jasper) # Bind inputs. + # Copy one input port definitions - using "user" port names. + Jasper.inputs["input"] = data_preprocessor.input_ports["input_signal"] + # Bind selected inputs - bind other using the default port name. + i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper.inputs["input"], length=Jasper) i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) - i_log_probs = jasper_decoder(encoder_output=i_encoded) # All output ports are bind (for now!) + i_log_probs = jasper_decoder(encoder_output=i_encoded) + # Bind selected outputs - using "user" port names. + Jasper.outputs["log_probs"] = i_log_probs + Jasper.outputs["encoded_len"] = i_encoded_len # Serialize graph serialized_jasper = Jasper.serialize() print("Serialized:\n", serialized_jasper) -# 'connections': -# * ['module1.processed_signal->module2.processed_signal', -# * 'module1.processed_length->module2.processed_length', -# * 'module2.outputs->module3.outputs'], -# 'inputs': -# * ['input_signal->module1.input_signal', -# * 'length->module1.length'], -# 'outputs': -# * {'outputs': ['module1.processed_signal->processed_signal', -# * 'module1.processed_length->processed_length', -# * 'module2.outputs->outputs', -# * 'module2.encoded_lengths->encoded_lengths', -# * 'module3.output->output'] - -# Delete everything - aside of jasper encoder, just as a test! ;) +# Delete everything - aside of jasper encoder, just as a test to show that reusing work! ;) del Jasper del data_preprocessor # del jasper_encoder # del jasper_decoder -# Deserialize graph. +# Deserialize graph - copy of the JASPER "model". jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") serialized_jasper_copy = jasper_copy.serialize() print("Deserialized:\n", serialized_jasper_copy) assert serialized_jasper == serialized_jasper_copy +# Create the "training" graph. with NeuralGraph(name="training") as training_graph: # Create the "implicit" training graph. o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() # Use Jasper module as any other neural module. - o_processed_signal, o_processed_signal_len, o_encoded, o_encoded_len, o_log_probs = jasper_copy( - input_signal=o_audio_signal, length=o_audio_signal_len - ) + o_log_probs, o_encoded_len = jasper_copy(input=o_audio_signal, length=o_audio_signal_len) o_predictions = greedy_decoder(log_probs=o_log_probs) o_loss = ctc_loss( log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len From 948f847940378124c437f53884295b5587a815dd Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 29 Apr 2020 17:12:02 -0700 Subject: [PATCH 085/106] first working example Signed-off-by: Jason --- examples/asr/jasper_an4_debug.py | 296 ++++++++++++++++++++ nemo/backends/pytorch/actions.py | 28 +- nemo/core/callbacks.py | 39 +++ nemo/core/neural_factory.py | 63 ++++- nemo/core/neural_types/__init__.py | 1 + nemo/core/neural_types/neural_type.py | 6 + nemo/core/neural_types/nmtensor_registry.py | 16 +- nemo/utils/app_state.py | 11 +- 8 files changed, 434 insertions(+), 26 deletions(-) create mode 100755 examples/asr/jasper_an4_debug.py diff --git a/examples/asr/jasper_an4_debug.py b/examples/asr/jasper_an4_debug.py new file mode 100755 index 000000000000..74b3e268f8d2 --- /dev/null +++ b/examples/asr/jasper_an4_debug.py @@ -0,0 +1,296 @@ +# Copyright (c) 2019 NVIDIA Corporation +import argparse +import math +import os +from functools import partial + +from ruamel.yaml import YAML + +import nemo +import nemo.collections.asr as nemo_asr +import nemo.utils.argparse as nm_argparse +from nemo.collections.asr.helpers import ( + monitor_asr_train_progress, + post_process_predictions, + post_process_transcripts, + process_evaluation_batch, + process_evaluation_epoch, + word_error_rate, +) +from nemo.utils.lr_policies import CosineAnnealing + +logging = nemo.logging + + +def create_dags(model_config_file, vocab, args, nf): + + # Create a data_layer for training. + data_layer = nemo_asr.AudioToTextDataLayer.import_from_config( + model_config_file, + "AudioToTextDataLayer_train", + overwrite_params={"manifest_filepath": args.train_dataset, "batch_size": args.batch_size}, + ) + + num_samples = len(data_layer) + steps_per_epoch = math.ceil(num_samples / (data_layer.batch_size * args.iter_per_step * nf.world_size)) + total_steps = steps_per_epoch * args.num_epochs + logging.info("Train samples=", num_samples, "num_steps=", total_steps) + + # # Create a data_layer for evaluation. + # data_layer_eval = nemo_asr.AudioToTextDataLayer.import_from_config( + # model_config_file, "AudioToTextDataLayer_eval", overwrite_params={"manifest_filepath": args.eval_datasets}, + # ) + + # num_samples = len(data_layer_eval) + # logging.info(f"Eval samples={num_samples}") + + # Instantiate data processor. + data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.import_from_config( + model_config_file, "AudioToMelSpectrogramPreprocessor" + ) + + # Instantiate JASPER encoder-decoder modules. + jasper_encoder = nemo_asr.JasperEncoder.import_from_config(model_config_file, "JasperEncoder") + jasper_decoder = nemo_asr.JasperDecoderForCTC.import_from_config( + model_config_file, "JasperDecoderForCTC", overwrite_params={"num_classes": len(vocab)} + ) + + # Instantiate losses. + ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) + greedy_decoder = nemo_asr.GreedyCTCDecoder() + + # Create a training graph. + audio, audio_len, transcript, transcript_len = data_layer() + processed, processed_len = data_preprocessor(input_signal=audio, length=audio_len) + encoded, encoded_len = jasper_encoder(audio_signal=processed, length=processed_len) + log_probs = jasper_decoder(encoder_output=encoded) + predictions = greedy_decoder(log_probs=log_probs) + loss = ctc_loss(log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len,) + + # # Create an evaluation graph. + # audio_e, audio_len_e, transcript_e, transcript_len_e = data_layer_eval() + # processed_e, processed_len_e = data_preprocessor(input_signal=audio_e, length=audio_len_e) + # encoded_e, encoded_len_e = jasper_encoder(audio_signal=processed_e, length=processed_len_e) + # log_probs_e = jasper_decoder(encoder_output=encoded_e) + # predictions_e = greedy_decoder(log_probs=log_probs_e) + # loss_e = ctc_loss( + # log_probs=log_probs_e, targets=transcript_e, input_length=encoded_len_e, target_length=transcript_len_e, + # ) + logging.info("Num of params in encoder: {0}".format(jasper_encoder.num_weights)) + + # Callbacks to print info to console and Tensorboard. + # train_callback = nemo.core.SimpleLossLoggerCallback( + # tensors=[loss, predictions, transcript, transcript_len], + # print_func=partial(monitor_asr_train_progress, labels=vocab), + # get_tb_values=lambda x: [["loss", x[0]]], + # tb_writer=nf.tb_writer, + # ) + + loss.rename("test") + train_callback = nemo.core.SimpleLossLogger(tensors_to_log=["test"]) + + # checkpointer_callback = nemo.core.CheckpointCallback(folder=nf.checkpoint_dir, step_freq=args.checkpoint_save_freq) + + # eval_tensors = [loss_e, predictions_e, transcript_e, transcript_len_e] + # eval_callback = nemo.core.EvaluatorCallback( + # eval_tensors=eval_tensors, + # user_iter_callback=partial(process_evaluation_batch, labels=vocab), + # user_epochs_done_callback=process_evaluation_epoch, + # eval_step=args.eval_freq, + # tb_writer=nf.tb_writer, + # eval_at_start=not args.do_not_eval_at_start, + # ) + # callbacks = [train_callback, checkpointer_callback, eval_callback] + callbacks = [train_callback] + + # Return entities required by the actual training. + return ( + loss, + # eval_tensors, + callbacks, + total_steps, + # log_probs_e, + # encoded_len_e, + ) + + +def main(): + parser = argparse.ArgumentParser( + parents=[nm_argparse.NemoArgParser()], description='AN4 ASR', conflict_handler='resolve', + ) + + # Overwrite default args + parser.add_argument("--train_dataset", type=str, help="training dataset path") + parser.add_argument("--eval_datasets", type=str, help="validation dataset path") + + # Create new args + # parser.add_argument("--lm", default="./an4-lm.3gram.binary", type=str) + parser.add_argument("--batch_size", default=48, type=int, help="size of the training batch") + parser.add_argument("--lm", default=None, type=str) + parser.add_argument("--test_after_training", action='store_true') + parser.add_argument("--momentum", type=float) + parser.add_argument("--beta1", default=0.95, type=float) + parser.add_argument("--beta2", default=0.25, type=float) + parser.add_argument("--do_not_eval_at_start", action='store_true') + parser.set_defaults( + model_config="./configs/jasper_an4.yaml", + train_dataset="~/TestData/an4_dataset/an4_train.json", + eval_datasets="~/TestData/an4_dataset/an4_val.json", + work_dir="./tmp", + optimizer="novograd", + num_epochs=50, + lr=0.02, + weight_decay=0.005, + checkpoint_save_freq=1000, + eval_freq=100, + amp_opt_level="O1", + ) + + args = parser.parse_args() + betas = (args.beta1, args.beta2) + + wer_thr = 0.20 + beam_wer_thr = 0.15 + + nf = nemo.core.NeuralModuleFactory( + local_rank=args.local_rank, + files_to_copy=[__file__], + optimization_level=args.amp_opt_level, + random_seed=0, + log_dir=args.work_dir, + create_tb_writer=True, + cudnn_benchmark=args.cudnn_benchmark, + ) + tb_writer = nf.tb_writer + checkpoint_dir = nf.checkpoint_dir + + # Load model definition + yaml = YAML(typ="safe") + with open(args.model_config) as f: + jasper_params = yaml.load(f) + # Get vocabulary. + vocab = jasper_params['labels'] + + # (loss, eval_tensors, callbacks, total_steps, log_probs_e, encoded_len_e,) = create_dags( + # args.model_config, vocab, args, nf + # ) + + loss, callbacks, total_steps = create_dags(args.model_config, vocab, args, nf) + + nf.train( + tensors_to_optimize=[loss], + callbacks=callbacks, + optimizer=args.optimizer, + lr_policy=CosineAnnealing(total_steps=total_steps, min_lr=args.lr / 100), + optimization_params={ + "num_epochs": args.num_epochs, + "max_steps": args.max_steps, + "lr": args.lr, + "momentum": args.momentum, + "betas": betas, + "weight_decay": args.weight_decay, + "grad_norm_clip": None, + }, + batches_per_step=args.iter_per_step, + amp_max_loss_scale=256.0, + # synced_batchnorm=(nf.global_rank is not None), + ) + + # if args.test_after_training: + # logging.info("Testing greedy and beam search with LM WER.") + # # Create BeamSearch NM + # if nf.world_size > 1 or args.lm is None: + # logging.warning("Skipping beam search WER as it does not work if doing distributed training.") + # else: + # beam_search_with_lm = nemo_asr.BeamSearchDecoderWithLM( + # vocab=vocab, beam_width=64, alpha=2.0, beta=1.5, lm_path=args.lm, num_cpus=max(os.cpu_count(), 1), + # ) + # beam_predictions = beam_search_with_lm(log_probs=log_probs_e, log_probs_length=encoded_len_e) + # eval_tensors.append(beam_predictions) + + # evaluated_tensors = nf.infer(eval_tensors) + # if nf.global_rank in [0, None]: + # greedy_hypotheses = post_process_predictions(evaluated_tensors[1], vocab) + # references = post_process_transcripts(evaluated_tensors[2], evaluated_tensors[3], vocab) + # wer = word_error_rate(hypotheses=greedy_hypotheses, references=references) + # logging.info("Greedy WER: {:.2f}%".format(wer * 100)) + # if wer > wer_thr: + # nf.sync_all_processes(False) + # raise ValueError(f"Final eval greedy WER {wer * 100:.2f}% > :" f"than {wer_thr * 100:.2f}%") + # nf.sync_all_processes() + + # if nf.world_size == 1 and args.lm is not None: + # beam_hypotheses = [] + # # Over mini-batch + # for i in evaluated_tensors[-1]: + # # Over samples + # for j in i: + # beam_hypotheses.append(j[0][1]) + + # beam_wer = word_error_rate(hypotheses=beam_hypotheses, references=references) + # logging.info("Beam WER {:.2f}%".format(beam_wer * 100)) + # assert beam_wer <= beam_wer_thr, "Final eval beam WER {:.2f}% > than {:.2f}%".format( + # beam_wer * 100, beam_wer_thr * 100 + # ) + # assert beam_wer <= wer, "Final eval beam WER > than the greedy WER." + + # # Reload model weights and train for extra 10 epochs + # checkpointer_callback = nemo.core.CheckpointCallback( + # folder=checkpoint_dir, step_freq=args.checkpoint_save_freq, force_load=True, + # ) + + # # Distributed Data Parallel changes the underlying class so we need + # # to reinstantiate Encoder and Decoder + # args.num_epochs += 10 + # previous_step_count = total_steps + # loss, eval_tensors, callbacks, total_steps, _, _ = create_dags(args.model_config, vocab, args, nf) + + # nf.reset_trainer() + # nf.train( + # tensors_to_optimize=[loss], + # callbacks=callbacks, + # optimizer=args.optimizer, + # lr_policy=CosineAnnealing(warmup_steps=previous_step_count, total_steps=total_steps), + # optimization_params={ + # "num_epochs": args.num_epochs, + # "lr": args.lr / 100, + # "momentum": args.momentum, + # "betas": betas, + # "weight_decay": args.weight_decay, + # "grad_norm_clip": None, + # }, + # reset=True, + # amp_max_loss_scale=256.0, + # # synced_batchnorm=(nf.global_rank is not None), + # ) + + # evaluated_tensors = nf.infer(eval_tensors) + # if nf.global_rank in [0, None]: + # greedy_hypotheses = post_process_predictions(evaluated_tensors[1], vocab) + # references = post_process_transcripts(evaluated_tensors[2], evaluated_tensors[3], vocab) + # wer_new = word_error_rate(hypotheses=greedy_hypotheses, references=references) + # logging.info("New greedy WER: {:.2f}%".format(wer_new * 100)) + # if wer_new > wer * 1.1: + # nf.sync_all_processes(False) + # raise ValueError( + # f"Fine tuning: new WER {wer_new * 100:.2f}% > than the " f"previous WER {wer * 100:.2f}%" + # ) + # nf.sync_all_processes() + + # # Open the log file and ensure that epochs is strictly increasing + # if nf._exp_manager.log_file: + # epochs = [] + # with open(nf._exp_manager.log_file, "r") as log_file: + # line = log_file.readline() + # while line: + # index = line.find("Starting epoch") + # if index != -1: + # epochs.append(int(line[index + len("Starting epoch") :])) + # line = log_file.readline() + # for i, e in enumerate(epochs): + # if i != e: + # raise ValueError("Epochs from logfile was not understood") + + +if __name__ == "__main__": + main() diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index ec24eb97362f..37197c471d59 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -20,8 +20,8 @@ from nemo.backends.pytorch.nm import DataLayerNM, TrainableNM from nemo.backends.pytorch.optimizers import AdamW, Novograd, master_params from nemo.core import DeploymentFormat, DeviceType, NeuralModule, NmTensor -from nemo.core.callbacks import ActionCallback, EvaluatorCallback, SimpleLossLoggerCallback -from nemo.core.neural_factory import Actions, OperationMode, Optimization +from nemo.core.callbacks import ActionCallback, EvaluatorCallback, SimpleLossLoggerCallback, NeMoCallback +from nemo.core.neural_factory import Actions, OperationMode, Optimization, TrainingState from nemo.core.neural_types import * from nemo.utils.helpers import get_checkpoint_from_dir @@ -450,10 +450,10 @@ def __nm_graph_forward_pass( if nm_tensor is None: continue t_name = nm_tensor.unique_name - if t_name not in registered_tensors: + if t_name not in registered_tensors or registered_tensors[t_name] is None: registered_tensors[t_name] = t_tensor else: - raise ValueError("A NMTensor was produced twice in " f"the same DAG. {t_name}") + raise ValueError(f"A NMTensor was produced twice in the same DAG. {t_name}") @staticmethod def pad_tensor(t: torch.Tensor, target_size: torch.Size): @@ -1101,6 +1101,7 @@ def train( gradient_predivide=False, amp_max_loss_scale=2.0 ** 24, ): + self._training_state = TrainingState() # Analyse the arguments passed to train. if tensors_to_optimize is not None and training_graph is not None: raise ValueError("Cannot pass both `tensors_to_optimize` and `training_graph` to the train() function") @@ -1195,7 +1196,7 @@ def train( # callbacks setup if callbacks is not None: for callback in callbacks: - if not isinstance(callback, ActionCallback): + if not isinstance(callback, ActionCallback) and not isinstance(callback, NeMoCallback): raise ValueError("A callback was received that was not a child of ActionCallback") elif isinstance(callback, SimpleLossLoggerCallback): if logging_callchain: @@ -1389,20 +1390,20 @@ def train( else: tensors.append(d) - registered_tensors = { - t.unique_name: d for t, d in zip(curr_call_chain[0][2].values(), tensors) if t is not None - } + for t, d in zip(curr_call_chain[0][2].values(), tensors): + if t is not None: + self.training_state.set_tensor(t, d) disable_allreduce = batch_counter < (batches_per_step - 1) self.__nm_graph_forward_pass( - call_chain=curr_call_chain, registered_tensors=registered_tensors, + call_chain=curr_call_chain, registered_tensors=self.training_state.tensor_dict, ) curr_tensors_to_optimize = training_loop[self.step % len(training_loop)][1] final_loss = 0 for tensor in curr_tensors_to_optimize: if ( - torch.isnan(registered_tensors[tensor.unique_name]).any() - or torch.isinf(registered_tensors[tensor.unique_name]).any() + torch.isnan(self.training_state.tensor_dict[tensor.unique_name]).any() + or torch.isinf(self.training_state.tensor_dict[tensor.unique_name]).any() ): if ( (stop_on_nan_loss) @@ -1418,7 +1419,7 @@ def train( ) else: logging.warning('Loss is NaN or inf, continuing training') - final_loss += registered_tensors[tensor.unique_name] + final_loss += self.training_state.tensor_dict[tensor.unique_name] if self._optim_level in AmpOptimizations and self._optim_level != Optimization.mxprO0: with amp.scale_loss(final_loss, curr_optimizer, delay_unscale=disable_allreduce) as scaled_loss: @@ -1461,10 +1462,11 @@ def train( batch_counter = 0 # Register iteration end with callbacks self._update_callbacks( - callbacks=callbacks, registered_tensors=registered_tensors, + callbacks=callbacks, registered_tensors=self.training_state.tensor_dict, ) self._perform_on_iteration_end(callbacks=callbacks) self.step += 1 + self.training_state.clear_dict() # End of epoch for loop # Register epochs end with callbacks self._perform_on_epoch_end(callbacks=callbacks) diff --git a/nemo/core/callbacks.py b/nemo/core/callbacks.py index e465bf5bf95a..2aafc6917ef3 100644 --- a/nemo/core/callbacks.py +++ b/nemo/core/callbacks.py @@ -37,6 +37,45 @@ logging = nemo.logging +class NeMoCallback(ABC): + def on_action_start(self, state): + pass + + def on_action_end(self, state): + pass + + def on_epoch_start(self, state): + pass + + def on_epoch_end(self, state): + pass + + def on_iteration_start(self, state): + pass + + def on_iteration_end(self, state): + pass + + +class SimpleLossLogger(NeMoCallback): + def __init__(self, step_freq=100, tensors_to_log=["loss"]): + #Step_freq: how often logs are printed + self.step_freq = step_freq + self.tensors_to_log = tensors_to_log + + # def on_optimizer_step_stop(self, state, tensors_to_log=[“loss”]): + # #tensors_to_log: List of keys into state that will be logged + + def on_iteration_end(self, state): + if state["step"] % self.step_freq == 0: + for tensor_key in self.tensors_to_log: + tensor = state["tensors"].get_tensor(tensor_key) + logging.info("%s: %s", tensor_key, tensor) + # except KeyError: + # raise KeyError(f"{self} was passed {tensor_key} but the tensor was not found in the state_dict. " + # f"Current state tensors include {state['tensors'].tensor_list()}") + + class ActionCallback(ABC): """Abstract interface for callbacks. """ diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 8aecda3d64d9..5636aef3a1fe 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -36,6 +36,7 @@ from ..utils import ExpManager from .callbacks import ActionCallback, EvaluatorCallback from .neural_types import * +from nemo.utils.app_state import AppState from nemo.utils.decorators import deprecated logging = nemo.logging @@ -84,6 +85,26 @@ class DeviceType(Enum): AllGpu = 3 +class TrainingState(): + def __init__(self): + tensor_naming_registery = AppState().tensor_names + self.tensor_dict = {}.fromkeys(tensor_naming_registery.unique_names, None) + + def tensor_list(self): + return self.tensor_dict.keys() + + def clear_dict(self): + for name in self.tensor_dict: + self.tensor_dict[name] = None + + def set_tensor(self, tensor, value): + self.tensor_dict[tensor.unique_name] = value + + def get_tensor(self, name): + unique_name = AppState().tensor_names[name] + return self.tensor_dict[unique_name] + + class Actions(ABC): """Basic actions allowed on graphs of Neural Modules""" @@ -93,6 +114,15 @@ def __init__(self, local_rank, global_rank, optimization_level=Optimization.mxpr self._optim_level = optimization_level self.step = None self.epoch_num = None + self._training_state = TrainingState() + + @property + def state(self): + return {"step": self.step, "tensors": self.training_state} + + @property + def training_state(self): + return self._training_state @property def local_rank(self): @@ -201,37 +231,56 @@ def _perform_on_iteration_start(self, callbacks): # to be a list of ActionCallback objects if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_iteration_start() + if isinstance(callback, ActionCallback): + callback.on_iteration_start() + else: + callback.on_iteration_start(self.state) def _perform_on_iteration_end(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_iteration_end() + if isinstance(callback, ActionCallback): + callback.on_iteration_end() + else: + callback.on_iteration_end(self.state) def _perform_on_action_start(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_action_start() + if isinstance(callback, ActionCallback): + callback.on_action_start() + else: + callback.on_action_start(self.state) def _perform_on_action_end(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_action_end() + if isinstance(callback, ActionCallback): + callback.on_action_end() + else: + callback.on_action_end(self.state) def _perform_on_epoch_start(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_epoch_start() + if isinstance(callback, ActionCallback): + callback.on_epoch_start() + else: + callback.on_epoch_start(self.state) def _perform_on_epoch_end(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.on_epoch_end() + if isinstance(callback, ActionCallback): + callback.on_epoch_end() + else: + callback.on_epoch_end(self.state) def _init_callbacks(self, callbacks): if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback.action = self + if isinstance(callback, ActionCallback): + callback.action = self def _update_callbacks( self, callbacks=None, registered_tensors=None, diff --git a/nemo/core/neural_types/__init__.py b/nemo/core/neural_types/__init__.py index 1fb5bf349076..0ae947d90137 100644 --- a/nemo/core/neural_types/__init__.py +++ b/nemo/core/neural_types/__init__.py @@ -19,3 +19,4 @@ from nemo.core.neural_types.comparison import * from nemo.core.neural_types.elements import * from nemo.core.neural_types.neural_type import * +from nemo.core.neural_types.nmtensor_registry import NmTensorNameRegistry diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 382215fcb59e..09f01943a322 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -223,6 +223,7 @@ def __init__(self, producer, producer_args, output_port_name, ntype=None): self._step_number = AppState().active_graph.step_number # List of tuples (step number, module name, input port name) self._consumers = [] + AppState().tensor_names.register(self) @property def producer(self): @@ -323,6 +324,11 @@ def unique_name(self): raise ValueError("This NmTensor does not have a unique name") return f"{self._output_port_name}~~~{self._producer_name}~~~{self._uuid}" + def rename(self, new_name): + """TODO + """ + AppState().tensor_names.rename_NmTensor(self, new_name) + class NeuralTypeError(Exception): """Base class for neural type related exceptions.""" diff --git a/nemo/core/neural_types/nmtensor_registry.py b/nemo/core/neural_types/nmtensor_registry.py index f9246cf128ec..2eb20eb81c07 100755 --- a/nemo/core/neural_types/nmtensor_registry.py +++ b/nemo/core/neural_types/nmtensor_registry.py @@ -24,7 +24,7 @@ def __init__(self): """ # Create the nmtensor_naming_dict # which contains a mapping of str to NMTensor.unique_name - self._nmtensor_naming_dict = {"loss": None} # Reserve keyname of 'loss' + self._nmtensor_naming_dict = {"loss": "loss"} # Reserve keyname of 'loss' self._nmtensor_uniname_set = set() # def summary(self): @@ -34,7 +34,12 @@ def __init__(self): # desc = desc + "`{}`: {}\n".format(graph.name, graph) # return desc - def register(self, tensor: NmTensor): + @property + def unique_names(self): + return self._nmtensor_uniname_set + + # def register(self, tensor: NmTensor): + def register(self, tensor): """TODO """ @@ -45,12 +50,13 @@ def register(self, tensor: NmTensor): # Finally, add object to the set. self._nmtensor_uniname_set.add(tensor.unique_name) - def rename_NmTensor(self, tensor: NmTensor, new_name: str): + # def rename_NmTensor(self, tensor: NmTensor, new_name: str): + def rename_NmTensor(self, tensor, new_name: str): """ TODO """ # Find old name if exists old_name = tensor.unique_name - for custom_name, unique_name in self._nmtensor_naming_dict: + for custom_name, unique_name in self._nmtensor_naming_dict.items(): if unique_name == tensor.unique_name: old_name = custom_name @@ -59,7 +65,7 @@ def rename_NmTensor(self, tensor: NmTensor, new_name: str): if new_name in self._nmtensor_naming_dict: raise KeyError(f"{new_name} already exists in current graph. Please use a unique name") - self._nmtensor_naming_dict["new_name"] = tensor.unique_name + self._nmtensor_naming_dict[new_name] = tensor.unique_name def __getitem__(self, key): """ diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index 76da9e36065d..e58a1e10ad1a 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -44,7 +44,16 @@ def __init__(self, device=None): # Create graph manager (registry with some additional functionality). self._neural_graph_manager = nemo.core.NeuralGraphManager() # Create NmTensor registry - self._module_registry = nemo.core.neural_types.NmTensorNameRegistry() + self._nmtensor_name_registry = nemo.core.neural_types.NmTensorNameRegistry() + + @property + def tensor_names(self): + """ Property returning the existing modules. + + Returns: + Existing modules (a set object). + """ + return self._nmtensor_name_registry @property def modules(self): From a98c19010ec0a4969b4e3d7623204807b850451b Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 29 Apr 2020 17:23:59 -0700 Subject: [PATCH 086/106] fix style Signed-off-by: Jason --- nemo/backends/pytorch/actions.py | 2 +- nemo/core/callbacks.py | 2 +- nemo/core/neural_factory.py | 2 +- nemo/core/neural_types/nmtensor_registry.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 37197c471d59..94956d59e956 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -20,7 +20,7 @@ from nemo.backends.pytorch.nm import DataLayerNM, TrainableNM from nemo.backends.pytorch.optimizers import AdamW, Novograd, master_params from nemo.core import DeploymentFormat, DeviceType, NeuralModule, NmTensor -from nemo.core.callbacks import ActionCallback, EvaluatorCallback, SimpleLossLoggerCallback, NeMoCallback +from nemo.core.callbacks import ActionCallback, EvaluatorCallback, NeMoCallback, SimpleLossLoggerCallback from nemo.core.neural_factory import Actions, OperationMode, Optimization, TrainingState from nemo.core.neural_types import * from nemo.utils.helpers import get_checkpoint_from_dir diff --git a/nemo/core/callbacks.py b/nemo/core/callbacks.py index 2aafc6917ef3..1161cef57ee2 100644 --- a/nemo/core/callbacks.py +++ b/nemo/core/callbacks.py @@ -59,7 +59,7 @@ def on_iteration_end(self, state): class SimpleLossLogger(NeMoCallback): def __init__(self, step_freq=100, tensors_to_log=["loss"]): - #Step_freq: how often logs are printed + # Step_freq: how often logs are printed self.step_freq = step_freq self.tensors_to_log = tensors_to_log diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 5636aef3a1fe..10397daa351c 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -85,7 +85,7 @@ class DeviceType(Enum): AllGpu = 3 -class TrainingState(): +class TrainingState: def __init__(self): tensor_naming_registery = AppState().tensor_names self.tensor_dict = {}.fromkeys(tensor_naming_registery.unique_names, None) diff --git a/nemo/core/neural_types/nmtensor_registry.py b/nemo/core/neural_types/nmtensor_registry.py index 2eb20eb81c07..a77e53f54bab 100755 --- a/nemo/core/neural_types/nmtensor_registry.py +++ b/nemo/core/neural_types/nmtensor_registry.py @@ -15,7 +15,7 @@ # ============================================================================= -class NmTensorNameRegistry(): +class NmTensorNameRegistry: def __init__(self): """ Constructor. Initializes the manager. Sets active graph to None. From ad5ba14371946495b2b417178e12e5140e3ac6a7 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 30 Apr 2020 16:42:48 -0700 Subject: [PATCH 087/106] add 'loss' to state Signed-off-by: Jason --- examples/asr/jasper_an4_debug.py | 6 ++++-- nemo/backends/pytorch/actions.py | 2 +- nemo/core/neural_factory.py | 7 +++++-- nemo/core/neural_types/nmtensor_registry.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/asr/jasper_an4_debug.py b/examples/asr/jasper_an4_debug.py index 74b3e268f8d2..e19ea0117f62 100755 --- a/examples/asr/jasper_an4_debug.py +++ b/examples/asr/jasper_an4_debug.py @@ -86,8 +86,10 @@ def create_dags(model_config_file, vocab, args, nf): # tb_writer=nf.tb_writer, # ) - loss.rename("test") - train_callback = nemo.core.SimpleLossLogger(tensors_to_log=["test"]) + # loss.rename("test") + # train_callback = nemo.core.SimpleLossLogger(tensors_to_log=["test"]) + + train_callback = nemo.core.SimpleLossLogger() # checkpointer_callback = nemo.core.CheckpointCallback(folder=nf.checkpoint_dir, step_freq=args.checkpoint_save_freq) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 94956d59e956..d997f3ba60c3 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -1462,7 +1462,7 @@ def train( batch_counter = 0 # Register iteration end with callbacks self._update_callbacks( - callbacks=callbacks, registered_tensors=self.training_state.tensor_dict, + callbacks=callbacks, registered_tensors=self.training_state.tensor_dict, final_loss=final_loss ) self._perform_on_iteration_end(callbacks=callbacks) self.step += 1 diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 10397daa351c..689a1bcf4b9d 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -283,12 +283,15 @@ def _init_callbacks(self, callbacks): callback.action = self def _update_callbacks( - self, callbacks=None, registered_tensors=None, + self, callbacks=None, registered_tensors=None, final_loss=None, ): # if self.local_rank is None or self.local_rank == 0: if callbacks is not None and isinstance(callbacks, List) and len(callbacks) > 0: for callback in callbacks: - callback._registered_tensors = registered_tensors + if isinstance(callback, ActionCallback): + callback._registered_tensors = registered_tensors + else: # For now, we can use the old callback function. In the future we should improve this + self.training_state.tensor_dict["loss"] = final_loss def _str_to_opt_level(opt_str: str) -> Optimization: diff --git a/nemo/core/neural_types/nmtensor_registry.py b/nemo/core/neural_types/nmtensor_registry.py index a77e53f54bab..c439d4949c9d 100755 --- a/nemo/core/neural_types/nmtensor_registry.py +++ b/nemo/core/neural_types/nmtensor_registry.py @@ -25,7 +25,7 @@ def __init__(self): # Create the nmtensor_naming_dict # which contains a mapping of str to NMTensor.unique_name self._nmtensor_naming_dict = {"loss": "loss"} # Reserve keyname of 'loss' - self._nmtensor_uniname_set = set() + self._nmtensor_uniname_set = set(["loss"]) # def summary(self): # """ Prints a nice summary. """ From 53750554410a9a380a1120b964cf6622ec51d47b Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 17:02:56 -0700 Subject: [PATCH 088/106] PR polish, added type hints to all classes, added graph summary Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 10 +- .../graph_composition_integration_tests1.py | 5 +- nemo/core/neural_graph/__init__.py | 2 - nemo/core/neural_graph/graph_inputs.py | 80 +++++--- nemo/core/neural_graph/graph_outputs.py | 49 +++-- nemo/core/neural_graph/neural_graph.py | 174 +++++++++++------- .../core/neural_graph/neural_graph_manager.py | 19 +- nemo/core/neural_interface.py | 15 +- nemo/core/neural_modules.py | 54 +++--- nemo/core/neural_types/neural_type.py | 12 +- nemo/utils/__init__.py | 1 - nemo/utils/app_state.py | 15 +- nemo/utils/connection.py | 4 - nemo/utils/metaclasses.py | 6 +- nemo/utils/object_registry.py | 28 ++- 15 files changed, 286 insertions(+), 188 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 99e7a2fe5097..a58030f920c4 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -74,9 +74,12 @@ Jasper.outputs["log_probs"] = i_log_probs Jasper.outputs["encoded_len"] = i_encoded_len +# Print the summary. +logging.info(Jasper.summary()) + # Serialize graph serialized_jasper = Jasper.serialize() -print("Serialized:\n", serialized_jasper) +# print("Serialized:\n", serialized_jasper) # Delete everything - aside of jasper encoder, just as a test to show that reusing work! ;) del Jasper @@ -87,7 +90,7 @@ # Deserialize graph - copy of the JASPER "model". jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") serialized_jasper_copy = jasper_copy.serialize() -print("Deserialized:\n", serialized_jasper_copy) +# print("Deserialized:\n", serialized_jasper_copy) assert serialized_jasper == serialized_jasper_copy # Create the "training" graph. @@ -104,6 +107,9 @@ training_graph.outputs["o_loss"] = o_loss # training_graph.outputs["o_predictions"] = o_predictions # DOESN'T WORK?!? +# Print the summary. +logging.info(training_graph.summary()) + tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] train_callback = nemo.core.SimpleLossLoggerCallback( tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index e8e57ad114ac..b6addf4a0d90 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -35,8 +35,9 @@ lss = loss(predictions=p, target=t) # Manual bind. g0.output_ports["output"] = lss -# print(g0.output_ports) -# print(g0.output_ports["x"]) + +# Print the summary. +logging.info(g0.summary()) # SimpleLossLoggerCallback will print loss values to console. callback = SimpleLossLoggerCallback( diff --git a/nemo/core/neural_graph/__init__.py b/nemo/core/neural_graph/__init__.py index 4596cb39c10f..7575f45a6ad2 100644 --- a/nemo/core/neural_graph/__init__.py +++ b/nemo/core/neural_graph/__init__.py @@ -16,7 +16,5 @@ # limitations under the License. # ============================================================================= -from nemo.core.neural_graph.graph_inputs import GraphInput, GraphInputs -from nemo.core.neural_graph.graph_outputs import GraphOutput, GraphOutputs from nemo.core.neural_graph.neural_graph import * from nemo.core.neural_graph.neural_graph_manager import NeuralGraphManager diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index f0540ec6b29a..e3b811e8e378 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -17,6 +17,7 @@ # ============================================================================= from collections.abc import MutableMapping +from typing import Any, Dict, List, Optional, Union from nemo.core.neural_types import NeuralType from nemo.utils import logging @@ -26,19 +27,19 @@ class GraphInput(object): """ A helper class represenging a single bound input. """ - def __init__(self, ntype): + def __init__(self, ntype: NeuralType): """ Initializes object. Args: - type: a NeuralType object. + ntype: a NeuralType object. """ # (Neural) Type of input. self._ntype = ntype # List of StepModulePort tuples to which this input links to (step number, module name, port name). self._consumers = [] - def bind(self, step_module_ports): + def bind(self, step_module_ports: StepModulePort): """ Binds the (step-module-ports) to this "graph input". Args: @@ -52,12 +53,15 @@ def bind(self, step_module_ports): self._consumers.append(smp) @property - def ntype(self): - """ Returns NeuralType of that input. """ + def ntype(self) -> NeuralType: + """ + Returns: + NeuralType of a given input. + """ return self._ntype @property - def consumers(self): + def consumers(self) -> List[StepModulePort]: """ Returns: List of bound modules i.e. (step number, module name, port name) tupes. @@ -76,50 +80,72 @@ def __init__(self): """ self._inputs = {} - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Union[NeuralType, GraphInput]): """ This method is used to "create" a bound input, i.e. copy definition from indicated module input port. Args: key: name of the input port of the Neural Graph. - value: NeuralType that will be set. + value: NeuralType (or GraphInput) that will be set. + + Raises: + KeyError: Definition of a previously bound port is not allowed. + TypeError: Port definition must be must be a NeuralType or GraphInput type. """ - if key in self._inputs.keys(): - raise KeyError("Overwriting definition of a previously bound port `{}` is not allowed".format(key)) - # Make sure that a proper NeuralType definition was passed here. if isinstance(value, NeuralType): - val_type = value + ntype = value elif isinstance(value, GraphInput): - val_type = value.ntype + ntype = value.ntype else: raise TypeError("Port `{}` definition must be must be a NeuralType or GraphInput type".format(key)) - # Ok, add definition to list of mapped (module, port)s. - # Note: for now, there are no mapped modules, so copy only (neural) type. - self._inputs[key] = GraphInput(ntype=val_type) - def __getitem__(self, key): + if key in self._inputs.keys(): + if self._inputs[key].ntype == ntype: + raise KeyError("Overwriting definition of a previously bound port `{}` is not allowed".format(key)) + # Else: do nothing. + else: + # Ok, add definition to list of mapped (module, port)s. + # Note: for now, there are no mapped modules, so copy only the (neural) type. + self._inputs[key] = GraphInput(ntype=ntype) + + def __getitem__(self, key: str) -> GraphInput: """ Returns bound input. """ return self._inputs[key] - def __delitem__(self, key): - raise NotImplementedError("Deleting a bound input port is not allowed") + def __delitem__(self, key: str): + """ + Raises: + NotImplementedError as deletion of a bound input port is not allowed. + """ + raise NotImplementedError("Deletion of a bound input port is not allowed") def __iter__(self): - """ Iterates over the bound inputs. """ + """ + Returns: + Iterator over the dict of bound inputs. + """ return iter(self._inputs) - def __len__(self): - """ Return number of bound inputs. """ + def __len__(self) -> int: + """ + Return: + The number of bound inputs. + """ return len(self._inputs) @property - def definitions(self): - """ Property returns definitions of the input ports by extracting them on the fly from list. """ + def definitions(self) -> Dict[str, NeuralType]: + """ + Property returns definitions of the input ports by extracting them on the fly from list. + + Returns: + Dictionary of neural types associated with bound inputs. + """ # Extract port definitions (Neural Types) from the inputs list. return {k: v.ntype for k, v in self._inputs.items()} - def has_binding(self, step_number: int, port_name): + def has_binding(self, step_number: int, port_name: str) -> Optional[str]: """ Checks if there is a binding leading to a given step number (module) and its given port. (module name is redundant, thus skipped in this test). @@ -135,7 +161,7 @@ def has_binding(self, step_number: int, port_name): # Binding not found. return None - def serialize(self): + def serialize(self) -> List[str]: """ Method responsible for serialization of the graph inputs. Returns: @@ -155,7 +181,7 @@ def serialize(self): return serialized_inputs @classmethod - def deserialize(cls, serialized_inputs, modules): + def deserialize(cls, serialized_inputs: List[str], modules: Dict[str, 'NeuralModule']): """ Class method responsible for deserialization of graph inputs. diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 1184aed1b5d9..830c75772966 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -17,7 +17,9 @@ # ============================================================================= from collections.abc import MutableMapping +from typing import Any, Dict, List, Optional +from nemo.core.neural_types import NeuralType, NmTensor from nemo.utils import logging from nemo.utils.connection import StepModulePort @@ -25,24 +27,27 @@ class GraphOutput(object): """ A helper class represenging a single bound output. """ - def __init__(self, ntype, producer_step_module_port): + def __init__(self, ntype: NeuralType, producer_step_module_port: StepModulePort): """ Initializes object. Args: - type: a NeuralType object. + ntype: a NeuralType object. producer_step_module_port: a producer StepModulePort tuple (step number (module name), port name). """ self._ntype = ntype self._producer_step_module_port = producer_step_module_port @property - def ntype(self): - """ Returns NeuralType of that output. """ + def ntype(self) -> NeuralType: + """ + Returns: + NeuralType of a given output. + """ return self._ntype @property - def producer_step_module_port(self): + def producer_step_module_port(self) -> StepModulePort: """ Returns producer step port (step number (module), port name) tuple. """ return self._producer_step_module_port @@ -104,21 +109,28 @@ def __delitem__(self, key): raise NotImplementedError("Deleting a bound output is not allowed") def __iter__(self): - """ Iterates over the outputs - depending whether there are some manual outputs or not. """ + """ + Returns: + Iterator over the outputs - depending whether there are some manual outputs or not. + """ if len(self._manual_outputs) > 0: return iter(self._manual_outputs) else: # Use default dict. return iter(self._default_outputs) - def __len__(self): - """ Return number of outputs - depending whether there are some manual outputs or not. """ + def __len__(self) -> int: + """ + Returns: + The number of outputs - depending whether there are some manual outputs or not. + """ if len(self._manual_outputs) > 0: return len(self._manual_outputs) else: # Use default dict. return len(self._default_outputs) - def bind(self, tensors_ref, port_names=None): - """ Binds the default outputs. + def bind(self, tensors_ref: List[NmTensor], port_names: Optional[str] = None): + """ + Binds the "default" outputs. Args: tensors_ref: List of tensors to be added. @@ -145,8 +157,13 @@ def bind(self, tensors_ref, port_names=None): self._default_outputs[name] = GraphOutput(tensor.ntype, tensor.producer_step_module_port) @property - def definitions(self): - """ Property returns definitions of the output ports by extracting them on the fly from the bound outputs. """ + def definitions(self) -> Dict[str, GraphOutput]: + """ + Property returns definitions of the output ports by extracting them on the fly from the bound outputs. + + Returns: + Dictionary of neural types associated with bound outputs. + """ # Get the right output dictionary. d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs @@ -154,7 +171,7 @@ def definitions(self): return {k: v.ntype for k, v in d.items()} @property - def tensors(self): + def tensors(self) -> Dict[str, NmTensor]: """ Property returns output tensors by extracting them on the fly from the bound outputs. @@ -177,7 +194,7 @@ def tensors(self): return output_tensors @property - def tensor_list(self): + def tensor_list(self) -> List[NmTensor]: """ Property returns output tensors by extracting them on the fly from the bound outputs. @@ -200,7 +217,7 @@ def tensor_list(self): # Return the result. return output_tensor_list - def serialize(self): + def serialize(self) -> Dict[str, Any]: """ Method responsible for serialization of the graph outputs. Returns: @@ -228,7 +245,7 @@ def serialize(self): # Return the result. return serialized_outputs - def deserialize(self, serialized_outputs, modules): + def deserialize(self, serialized_outputs: Dict[str, Any], modules: Dict[str, 'NeuralModule']): """ Method responsible for deserialization of graph outputs. diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index bd45b769e24e..8ef51dd625bf 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -22,7 +22,7 @@ from collections import OrderedDict, namedtuple from os import path -from typing import Dict, Optional +from typing import Any, Dict, List, Optional, Union from ruamel.yaml import YAML @@ -31,7 +31,7 @@ from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_modules import NeuralModule -from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType +from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging from nemo.utils.connection import Connection, StepModulePort @@ -44,7 +44,7 @@ class NeuralGraph(NeuralInterface): Neural Graph class stores dynamically defined graphs of connected Neural Modules. """ - def __init__(self, operation_mode=OperationMode.both, name=None): + def __init__(self, operation_mode: OperationMode = OperationMode.both, name: Optional[str] = None): """ Constructor. Initializes graph variables. @@ -54,7 +54,7 @@ def __init__(self, operation_mode=OperationMode.both, name=None): name: Name of the graph (optional) """ # Initialize the inferface. - super().__init__(name) + super().__init__() # Register graph. self._name = self._app_state.register_graph(self, name) @@ -119,7 +119,7 @@ def __call__(self, **kwargs): # Return output tensors. return results - def nest(self, inner_graph, inner_graph_args): + def nest(self, inner_graph: 'NeuralGraph', inner_graph_args): """ Method nests (copies) a graph: modules, steps, topology (tensors). @@ -246,7 +246,7 @@ def nest(self, inner_graph, inner_graph_args): # Return the results. return results - def record_step(self, module): + def record_step(self, module: NeuralModule): """ Records the operation (module plus passed inputs) on a list. @@ -275,12 +275,14 @@ def record_step(self, module): return step_number @property - def step_number(self): - """ Returns: - Last step number. """ + def step_number(self) -> int: + """ + Returns: + The current step number. + """ return len(self._steps) - 1 - def bind_outputs(self, tensors_list): + def bind_outputs(self, tensors_list: Union[NmTensor, List[NmTensor]]): """ Binds the output tensors. @@ -307,7 +309,7 @@ def bind_outputs(self, tensors_list): self.outputs.bind(tensors_list) @property - def inputs(self): + def inputs(self) -> GraphInputs: """ Returns graph inputs. @@ -317,7 +319,7 @@ def inputs(self): return self._inputs @property - def input_ports(self) -> Optional[Dict[str, NeuralType]]: + def input_ports(self) -> Dict[str, NeuralType]: """ Returns definitions of graph input ports (dict of Neural Types). @@ -331,7 +333,7 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: return self._inputs.definitions @property - def outputs(self): + def outputs(self) -> GraphOutputs: """ Returns graph outputs. @@ -341,7 +343,7 @@ def outputs(self): return self._outputs @property - def output_ports(self) -> Optional[Dict[str, NeuralType]]: + def output_ports(self) -> Dict[str, NeuralType]: """ Returns definitions of module output ports (dict of Neural Types). @@ -356,7 +358,7 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: return self._outputs.definitions @property - def output_tensors(self): + def output_tensors(self) -> Dict[str, NmTensor]: """ Returns graph output tensors. @@ -366,26 +368,32 @@ def output_tensors(self): return self._outputs.tensors @property - def modules(self): + def modules(self) -> Dict[str, NeuralModule]: """ Returns modules. """ return self._modules - def __getitem__(self, key): + def __getitem__(self, key) -> NeuralModule: """ Returns module given its name (name of the variable). Args: key: Name of the variable. + + Raises: + KeyError: Neural Graph doesn't contain a module with a given name (key). """ if key not in self._modules.keys(): raise KeyError("Neural Graph doesn't contain a module named {}".format(key)) return self._modules[key] - def __len__(self): - """ Returns number of modules (vertices) in a given graph. """ + def __len__(self) -> int: + """ + Returns: + The number of modules (vertices) in a given graph. + """ return len(self._modules) @property - def steps(self): + def steps(self) -> Dict[int, str]: """ Returns steps. """ return self._steps @@ -401,7 +409,7 @@ def tensors(self): return self._all_tensors @property - def tensor_list(self): + def tensor_list(self) -> List[NmTensor]: """ Property returning output tensors by extracting them on the fly from the bound outputs. @@ -419,11 +427,14 @@ def tensor_list(self): return tensor_list @property - def operation_mode(self): - """ Returns operation mode. """ + def operation_mode(self) -> OperationMode: + """ + Returns: + Operation mode. + """ return self._operation_mode - def __enter__(self): + def __enter__(self) -> 'NeuralGraph': """ Activates this graph. @@ -451,8 +462,9 @@ def deactivate(self): """ self._app_state.active_graph = None - def export_to_config(self, config_file): - """ Exports the neural graph to a file. + def export_to_config(self, config_file: str): + """ + Exports the neural graph to a file. Args: config_file: Name (and path) of the config file (YML) to be written to. @@ -471,7 +483,7 @@ def export_to_config(self, config_file): "Configuration of graph `{}` ({}) exported to {}".format(self.name, type(self).__name__, abs_path_file) ) - def serialize(self): + def serialize(self) -> Dict[str, Any]: """ Method serializes the whole graph. Returns: @@ -501,7 +513,7 @@ def serialize(self): # Return the dictionary. return serialized_graph - def __serialize_header(self): + def __serialize_header(self) -> Dict[str, Any]: """ Private method responsible for serializing the graph header. Returns: @@ -520,7 +532,7 @@ def __serialize_header(self): # Return header. return header - def __serialize_modules(self): + def __serialize_modules(self) -> Dict[str, Any]: """ Private method responsible for serializing the modules present in the graph. Returns: @@ -542,7 +554,7 @@ def __serialize_steps(self): serialized_steps[no] = module_name return serialized_steps - def __serialize_connections(self): + def __serialize_connections(self) -> Dict[str, Any]: """ Private method responsible for serializing the connections in the graph. Returns: @@ -563,7 +575,13 @@ def __serialize_connections(self): return serialized_connections @classmethod - def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite_params={}, name=None): + def import_from_config( + cls, + config_file: str, + reuse_existing_modules: bool = False, + overwrite_params: Dict[str, Any] = {}, + name: Optional[str] = None, + ) -> 'NeuralGraph': """ Class method importing the neural graph from the configuration file. Raises an ImportError exception when config file is invalid. @@ -595,7 +613,7 @@ def import_from_config(cls, config_file, reuse_existing_modules=False, overwrite return new_graph @classmethod - def __validate_config_file(cls, config_file): + def __validate_config_file(cls, config_file: str): """ Class method validating whether the config file has a proper content (sections, specification etc.). Raises an ImportError exception when config file is invalid or @@ -636,7 +654,9 @@ def __validate_config_file(cls, config_file): return loaded_config @classmethod - def deserialize(cls, configuration, reuse_existing_modules=False, name=None): + def deserialize( + cls, configuration: Dict[str, Any], reuse_existing_modules: bool = False, name: Optional[str] = None + ) -> 'NeuralGraph': """ Class method creating a graph instance by deserializing the provided configuratino. @@ -681,7 +701,7 @@ def deserialize(cls, configuration, reuse_existing_modules=False, name=None): return new_graph @classmethod - def __deserialize_header(cls, serialized_header): + def __deserialize_header(cls, serialized_header: Dict[str, Any]): """ Private class method deserializing the header and extracts the general information. Args: @@ -704,14 +724,18 @@ def __deserialize_header(cls, serialized_header): # Return the mode. return operation_mode - def __deserialize_modules(self, serialized_modules, reuse_existing_modules): + def __deserialize_modules(self, serialized_modules: Dict[str, Any], reuse_existing_modules: bool): """ Private method deserializing the modules present in the graph. Args: serialized_modules: Dictionary containing graph modules. + reuse_existing_modules: If True, won create a new module when a module with a given name exists. Returns: Dictionary of modules. + + Raises: + KeyError: A module with name already exists (if reuse_existing_modules is set to False). """ modules = {} for name, module_params in serialized_modules.items(): @@ -728,7 +752,7 @@ def __deserialize_modules(self, serialized_modules, reuse_existing_modules): # Ok, done. return modules - def __deserialize_steps(self, serialized_steps): + def __deserialize_steps(self, serialized_steps: Dict[str, Any]): """ Private method deserializing the steps (order of module executions). Args: @@ -743,7 +767,7 @@ def __deserialize_steps(self, serialized_steps): # Ok, done. return steps - def __deserialize_connections(self, serialized_connections, modules): + def __deserialize_connections(self, serialized_connections: Dict[str, Any], modules: Dict[str, NeuralModule]): """ Private method deserializing the connections in the graph. Args: @@ -848,35 +872,51 @@ def __execute_and_create_tensors(self, steps, modules, connections, inputs): # Ok, now we can turn automatic binding on. self.default_output_binding = True - def summary(self): - """ Prints a nice summary. """ - # TODO: a nice summary. ;) - desc = "`{}` ({}):\n".format(self.name, len(self._steps)) - for num, op in self._steps.items(): - desc = desc + " {}. {}\n".format(num, type(op[0]).__name__) - return desc + def summary(self) -> str: + """ + Returns: + A nice, full graph summary. + """ + # Line "decorator". + desc = "\n" + 120 * '=' + "\n" + # 1. general information. + desc += "The `{}` Neural Graph:\n".format(self.name) + + # 2. modules. + desc += " * Modules ({}):\n".format(len(self._modules)) + for key, module in self._modules.items(): + desc += " * `{}` ({})\n".format(key, type(module).__name__) + + # 3. steps. + desc += " * Steps ({}):\n".format(len(self._steps)) + for num, module in self._steps.items(): + desc += " {}. {}\n".format(num, module) + + # 4. connections. + connections = self.__serialize_connections() + desc += " * Connections ({}):\n".format(len(connections)) + # if len(connections) == 0: + # desc += " -\n" + for connection in connections: + desc += " * {}\n".format(connection) + + # 5. graph (bound) inputs. + inputs = self._inputs.serialize() + desc += " * Graph Inputs ({}):\n".format(len(inputs)) + # if len(inputs) == 0: + # desc += " -\n" + for input in inputs: + desc += " * {}\n".format(input) + + # 6. graph (bound) outputs. + outputs = self._outputs.serialize() + desc += " * Graph Outputs ({}, {}):\n".format(len(outputs["mappings"]), outputs["type"]) + # if len(outputs) == 0: + # desc += " -\n" + for output in outputs["mappings"]: + desc += " * {}\n".format(output) + # Line "decorator". + desc += 120 * '=' + "\n" - def list_modules(self): - """ Lists modules. """ - desc = "{} ({}):\n".format(self.name, len(self._modules)) - for key, value in self._modules.items(): - desc += " * `{}` ({})\n".format(key, value) + # Return the result. return desc - - def show_inputs(self): - print("bound input ports: ") - # for key, value in self._bound_input_ports.items(): - # print(" * `{}`: `{}` ({})".format(key, value, type(value))) - - print("bound input tensors: ") - # for key, value in self._bound_input_tensors.items(): - # print(" * `{}`: `{}` ({})".format(key, value, type(value))) - - def show_outputs(self): - print("bound (default) output ports: ") - # for key, value in self._bound_output_ports_default.items(): - # print(" * `{}`: `{}` ({})".format(key, value, type(value))) - - print("bound (default) output tensors: ") - # for key, value in self._bound_output_tensors_default.items(): - # print(" * `{}`: `{}` ({})".format(key, value, type(value))) diff --git a/nemo/core/neural_graph/neural_graph_manager.py b/nemo/core/neural_graph/neural_graph_manager.py index 1e128b34fb5d..3c17216db78d 100644 --- a/nemo/core/neural_graph/neural_graph_manager.py +++ b/nemo/core/neural_graph/neural_graph_manager.py @@ -30,21 +30,28 @@ def __init__(self): self._active_graph = None def __eq__(self, other): - """ Checks if two managers have the same content. """ + """ + Checks if two managers have the same content. + Args: + other: A second manager object. + """ if not isinstance(other, ObjectRegistry): return False return super().__eq__(other) - def summary(self): - """ Prints a nice summary. """ + def summary(self) -> str: + """ + Returns: + A summary of the graphs on the list. + """ # TODO: a nicer summary. ;) - desc = "" + desc = "List of graphs:" for graph in self: desc = desc + "`{}`: {}\n".format(graph.name, graph) return desc @property - def active_graph(self): + def active_graph(self) -> NeuralGraph: """ Property returns the active graph. If there is no active graph, creates a new one. @@ -63,7 +70,7 @@ def active_graph(self): return self._active_graph @active_graph.setter - def active_graph(self, graph): + def active_graph(self, graph: NeuralGraph): """ Property sets the active graph. diff --git a/nemo/core/neural_interface.py b/nemo/core/neural_interface.py index 5605feb1bc67..9a374da9ef88 100644 --- a/nemo/core/neural_interface.py +++ b/nemo/core/neural_interface.py @@ -17,7 +17,7 @@ # ============================================================================= from abc import ABC, abstractmethod -from typing import Dict, Optional +from typing import Dict import nemo from nemo.core.neural_types import NeuralType @@ -33,17 +33,16 @@ class NeuralInterface(ABC): graph, e.g. get_weights, tie_weights, ) """ - def __init__(self, name): - """ Constructor. Sets the application state. """ - # Copy the name. As names should be unique in module/graph scope, this should be handled additionally - # in their constructors. - self._name = name + def __init__(self): + """ + Constructor. Creates a "shortcut" to the application state. + """ # Create access to the app state. self._app_state = nemo.utils.app_state.AppState() @property @abstractmethod - def input_ports(self) -> Optional[Dict[str, NeuralType]]: + def input_ports(self) -> Dict[str, NeuralType]: """ Returns definitions of module input ports Returns: @@ -52,7 +51,7 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: @property @abstractmethod - def output_ports(self) -> Optional[Dict[str, NeuralType]]: + def output_ports(self) -> Dict[str, NeuralType]: """ Returns definitions of module output ports Returns: diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 2bd71d9a6906..993861400a1c 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -24,7 +24,7 @@ from enum import Enum from inspect import getargvalues, getfullargspec, stack from os import path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from ruamel.yaml import YAML @@ -58,7 +58,7 @@ class NeuralModule(NeuralInterface): def __init__(self, name=None): # Initialize the inferface. - super().__init__(name) + super().__init__() # Retrieve dictionary of parameters (keys, values) passed to init. self._init_params = self.__extract_init_params() @@ -83,7 +83,7 @@ def __init__(self, name=None): self._opt_level = self._factory.optim_level @property - def init_params(self) -> Optional[Dict]: + def init_params(self) -> Dict[str, Any]: """ Property returning parameters used to instantiate the module. @@ -92,7 +92,7 @@ def init_params(self) -> Optional[Dict]: """ return self._init_params - def __extract_init_params(self): + def __extract_init_params(self) -> Dict[str, Any]: """ Retrieves the dictionary of of parameters (keys, values) passed to constructor of a class derived (also indirectly) from the Neural Module class. @@ -103,12 +103,6 @@ def __extract_init_params(self): # Get names of arguments of the original module init method. init_keys = getfullargspec(type(self).__init__).args - # (Pdb) localvars - # {'self': , 'batch_size': 1, 'f_name': 'sin', - # 'n': 100, 'x_lo': -4, 'x_hi': 4, 'name': 'tgs1_dl', - #'__class__': } - # Remove self. if "self" in init_keys: init_keys.remove("self") @@ -145,7 +139,7 @@ def __extract_init_params(self): # Return parameters. return init_params - def __validate_params(self, params): + def __validate_params(self, params: Dict[str, Any]) -> bool: """ Checks whether dictionary contains parameters being primitive types (string, int, float etc.) or (lists of)+ primitive types. @@ -171,7 +165,7 @@ def __validate_params(self, params): # Return the result. return ok - def __is_of_allowed_type(self, var): + def __is_of_allowed_type(self, var) -> bool: """ A recursive function that checks if a given variable is of allowed type. @@ -205,13 +199,15 @@ def __is_of_allowed_type(self, var): # Well, seems that everything is ok. return True - def export_to_config(self, config_file): + def export_to_config(self, config_file: str): """ A function that exports module "configuration" (i.e. init parameters) to a YAML file. - Raises a ValueError exception in case then parameters coudn't be exported. Args: config_file: path (absolute or relative) and name of the config file (YML) + + Raises: + ValueError: An error occurred and parameters coudn't be exported. """ # Greate an absolute path. abs_path_file = path.expanduser(config_file) @@ -227,7 +223,7 @@ def export_to_config(self, config_file): "Configuration of module `{}` ({}) exported to {}".format(self.name, type(self).__name__, abs_path_file) ) - def serialize(self): + def serialize(self) -> Dict[str, Any]: """ A method serializing the whole Neural module (into a dictionary). Returns: @@ -245,7 +241,7 @@ def serialize(self): # Return the dictionary. return serialized_module - def __serialize_header(self): + def __serialize_header(self) -> Dict[str, Any]: """ A protected method that creates a header stored later in the configuration file. Returns: @@ -292,7 +288,7 @@ def __serialize_header(self): } return header - def _serialize_configuration(self): + def _serialize_configuration(self) -> Dict[str, Any]: """ A function that serializes the module "configuration (i.e. init parameters) to a dictionary. Raises a ValueError exception in case then parameters coudn't be exported. @@ -301,7 +297,7 @@ def _serialize_configuration(self): Thus functions should be overloaded when writing a custom module import/export. Returns: - a "serialized" dictionary with module configuration. + A "serialized" dictionary with module configuration. """ # Check if generic export will work. if not self.__validate_params(self._init_params): @@ -314,7 +310,9 @@ def _serialize_configuration(self): return self._init_params @classmethod - def import_from_config(cls, config_file, section_name=None, name=None, overwrite_params={}): + def import_from_config( + cls, config_file: str, section_name: str = None, name: str = None, overwrite_params: Dict = {} + ) -> 'NeuralModule': """ Class method importing the configuration file. Raises an ImportError exception when config file is invalid or @@ -345,7 +343,7 @@ def import_from_config(cls, config_file, section_name=None, name=None, overwrite return obj @classmethod - def __validate_config_file(cls, config_file, section_name=None): + def __validate_config_file(cls, config_file: str, section_name: str = None) -> Dict[str, Any]: """ Class method validating whether the config file has a proper content (sections, specification etc.). Raises an ImportError exception when config file is invalid or @@ -399,7 +397,9 @@ def __validate_config_file(cls, config_file, section_name=None): return loaded_config @classmethod - def deserialize(cls, configuration, name=None, overwrite_params={}): + def deserialize( + cls, configuration: str, name: str = None, overwrite_params: Dict[str, Any] = {} + ) -> 'NeuralModule': """ Class method instantianting the neural module object based on the configuration (dictionary). @@ -441,7 +441,7 @@ def deserialize(cls, configuration, name=None, overwrite_params={}): return new_module @classmethod - def __deserialize_header(cls, serialized_header): + def __deserialize_header(cls, serialized_header: Dict[str, Any]): """ Method deserializes the header and extracts the module class. Args: @@ -462,7 +462,7 @@ def __deserialize_header(cls, serialized_header): return mod_obj @classmethod - def _deserialize_configuration(cls, serialized_init_params): + def _deserialize_configuration(cls, serialized_init_params: Dict[str, Any]): """ A function that deserializes the module "configuration (i.e. init parameters). @@ -489,7 +489,7 @@ def create_ports(**kwargs): @property @abstractmethod - def input_ports(self) -> Optional[Dict[str, NeuralType]]: + def input_ports(self) -> Dict[str, NeuralType]: """Returns definitions of module input ports Returns: @@ -498,7 +498,7 @@ def input_ports(self) -> Optional[Dict[str, NeuralType]]: @property @abstractmethod - def output_ports(self) -> Optional[Dict[str, NeuralType]]: + def output_ports(self) -> Dict[str, NeuralType]: """Returns definitions of module output ports Returns: @@ -506,7 +506,7 @@ def output_ports(self) -> Optional[Dict[str, NeuralType]]: """ @property - def _disabled_deployment_input_ports(self) -> Optional[Set[str]]: + def _disabled_deployment_input_ports(self) -> Set[str]: """Returns names of input ports that will not be included in an export Returns: @@ -515,7 +515,7 @@ def _disabled_deployment_input_ports(self) -> Optional[Set[str]]: return set([]) @property - def _disabled_deployment_output_ports(self) -> Optional[Set[str]]: + def _disabled_deployment_output_ports(self) -> Set[str]: """Returns names of output ports that will not be included in an export Returns: diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 5c4136d7abb4..3d72b1ee5d9f 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -23,7 +23,7 @@ 'NeuralPortNmTensorMismatchError', ] import uuid -from typing import Optional, Tuple +from typing import List, Optional, Tuple from nemo.core.neural_types.axes import AxisKind, AxisType from nemo.core.neural_types.comparison import NeuralTypeComparisonResult @@ -233,7 +233,7 @@ def producer(self): return AppState().modules[self._producer_name] @property - def producer_name(self): + def producer_name(self) -> str: """ Returns: Name of the producer of the tensor. @@ -241,7 +241,7 @@ def producer_name(self): return self._producer_name @property - def producer_step_number(self): + def producer_step_number(self) -> int: """ Returns: Step number indicating when the tensor was produced. @@ -250,7 +250,7 @@ def producer_step_number(self): return self._step_number @property - def producer_step_module_port(self): + def producer_step_module_port(self) -> StepModulePort: """ Returns: A tuple containing step number, module name and corresponding output port name. @@ -258,14 +258,14 @@ def producer_step_module_port(self): return StepModulePort(self._step_number, self._producer_name, self._output_port_name) @property - def consumers(self): + def consumers(self) -> List[StepModulePort]: """ Returns: A list of tuples containing consumer step number, module name and corresponding input port names. """ return self._consumers - def add_consumer(self, step_module_port): + def add_consumer(self, step_module_port: StepModulePort): """ Adds the "consumer" to tensor. diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 663d2a7e4cc0..d67c8ee31b77 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -25,4 +25,3 @@ from .helpers import * from nemo.utils.app_state import AppState from nemo.utils.object_registry import ObjectRegistry -from nemo.utils.connection import * diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index db7fc6ff48de..d47f4fa1a5dd 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -15,7 +15,9 @@ # limitations under the License. # ============================================================================= -# Sadly have to import this to avoid circular dependencies. +# Sadly have to import the whole "nemo" module to avoid circular dependencies. +# Moreover, at that point nemo module doesn't contain core, so during "python module registration" +# nothing from nemo.core, including types (so we cannot use them for "python 3 type hints"). import nemo from nemo.utils.metaclasses import Singleton @@ -40,13 +42,14 @@ def __init__(self, device=None): else: self._device = device # Create module registry. - self._module_registry = nemo.utils.ObjectRegistry("module") + self._module_registry = nemo.utils.object_registry.ObjectRegistry("module") # Create graph manager (registry with some additional functionality). self._neural_graph_manager = nemo.core.NeuralGraphManager() @property def modules(self): - """ Property returning the existing modules. + """ + Property returning the existing modules. Returns: Existing modules (a set object). @@ -62,7 +65,7 @@ def graphs(self): """ return self._neural_graph_manager - def register_module(self, module, name): + def register_module(self, module, name: str) -> str: """ Registers a module using the provided name. If name is none - generates a new unique name. @@ -76,7 +79,7 @@ def register_module(self, module, name): """ return self._module_registry.register(module, name) - def register_graph(self, graph, name): + def register_graph(self, graph, name: str) -> str: """ Registers a new graph using the provided name. If name is none - generates a new unique name. @@ -95,7 +98,7 @@ def active_graph(self): """ Property returns the active graph. Returns: - Active graph + Active graph. """ return self._neural_graph_manager.active_graph diff --git a/nemo/utils/connection.py b/nemo/utils/connection.py index ada4e0c0dbb7..e181f54d9876 100644 --- a/nemo/utils/connection.py +++ b/nemo/utils/connection.py @@ -16,10 +16,6 @@ # limitations under the License. # ============================================================================= -__all__ = [ - 'StepModulePort', - 'Connection', -] from collections import namedtuple diff --git a/nemo/utils/metaclasses.py b/nemo/utils/metaclasses.py index 8aed2d240e00..ead4b4560531 100644 --- a/nemo/utils/metaclasses.py +++ b/nemo/utils/metaclasses.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License.**** -__all__ = [ - "Singleton", -] - import threading @@ -30,7 +26,7 @@ class Singleton(type): __lock = threading.Lock() def __call__(cls, *args, **kwargs): - """ Returns singleton instance.A thread safe implementation. """ + """ Returns singleton instance. A thread safe implementation. """ if cls not in cls.__instances: # Enter critical section. with cls.__lock: diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index 2b7f068c3cb6..aed46167eb9e 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -30,14 +30,14 @@ def __init__(self, base_type_name): super().__init__() self._base_type_name = base_type_name - def register(self, new_obj, name): + def register(self, new_obj, name: str) -> str: """ Registers a new object using the provided name. If name is none - generates new unique name. Args: new_obj: An object to be registered. - name: A "proposition" of object name. + name: A "proposition" for the object name. Returns: A unique name (proposition or newly generated name). @@ -65,15 +65,20 @@ def register(self, new_obj, name): # Return the name. return unique_name - def has(self, name): - """ Check if registry stores object with a given name. """ + def has(self, name: str) -> bool: + """ + Check if registry stores object with a given name. + + Args: + name: name of the object to be found in the registry. + """ for obj in self: if obj.name == name: return True # Else: return False - def __generate_unique_name(self): + def __generate_unique_name(self) -> str: """ Generates a new unique name by adding postfix (number) to base name. @@ -93,7 +98,7 @@ def __generate_unique_name(self): postfix += 1 return new_name - def __getitem__(self, key): + def __getitem__(self, key: str): """ Object getter function. @@ -112,17 +117,22 @@ def __getitem__(self, key): raise KeyError("A {} with name `{}` don't exists!".format(self._base_type_name, key)) def __eq__(self, other): - """ Checks if two registers have the same content. """ + """ + Checks if two registers have the same content. + + Args: + other: The second registry object. + """ if not isinstance(other, WeakSet): return False return super().__eq__(other) - def summary(self): + def summary(self) -> str: """ Returns: A summary of the objects on the list. """ - summary = "Objects:\n" + summary = "Registry of {}s:\n".format(self._base_type_name) for obj in self: summary += " * {} ({})\n".format(obj.name, type(obj).__name__) return summary From 9f6ee98ab9458c9299a321763eeffcff200af94d Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 17:06:12 -0700 Subject: [PATCH 089/106] removed 'name' in for in actions.py Signed-off-by: Tomasz Kornuta --- nemo/backends/pytorch/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index ec24eb97362f..b6c71b37a03a 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -165,7 +165,7 @@ def is_in_degree_zero(node, processed_nodes): all_nodes[node][nmtensor.name] = nmtensor processed_nmtensors.add(nmtensor) if nmtensor.producer_args is not None and nmtensor.producer_args != {}: - for name, new_nmtensor in nmtensor.producer_args.items(): + for _, new_nmtensor in nmtensor.producer_args.items(): if new_nmtensor not in processed_nmtensors: # put in the start of list hooks_lst.insert(0, new_nmtensor) From 45dacd378dcbf36fdf0a3086a00f33ebd17a4812 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 17:16:15 -0700 Subject: [PATCH 090/106] NG-related integration tests Signed-off-by: Tomasz Kornuta --- .../core/test_integration_neural_graph.py | 41 ++++---- .../test_integration_neural_graph_nesting.py | 97 ------------------- 2 files changed, 19 insertions(+), 119 deletions(-) delete mode 100644 tests/integration/core/test_integration_neural_graph_nesting.py diff --git a/tests/integration/core/test_integration_neural_graph.py b/tests/integration/core/test_integration_neural_graph.py index 210921c5f848..8b85e62a0de7 100644 --- a/tests/integration/core/test_integration_neural_graph.py +++ b/tests/integration/core/test_integration_neural_graph.py @@ -24,28 +24,13 @@ @pytest.mark.usefixtures("neural_factory") -class TestNeuralGraph: - @pytest.mark.integration - def test_implicit_default_graph(self): - """ Tests integration of a `default` (implicit) graph. """ - # Create modules. - dl = RealFunctionDataLayer(n=100, batch_size=4) - fx = TaylorNet(dim=4) - loss = MSELoss() - - # This will create a default (implicit) graph: "training". - x, t = dl() - p = fx(x=x) - lss = loss(predictions=p, target=t) - - # Instantiate an optimizer to perform the `train` action. - optimizer = PtActions() - # Invoke "train" action - perform single forward-backard step. - optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") - +class TestNeuralGraphTrainAction: @pytest.mark.integration def test_explicit_graph(self): - """ Tests integration of an `explicit` graph and decoupling of graph creation from its activation. """ + """ + Tests the integration of an `explicit` graph and decoupling of graph creation from its activation. + Additionally checks whether user can pass NG instance to train(). + """ # Create modules. dl = RealFunctionDataLayer(n=100, batch_size=4) fx = TaylorNet(dim=4) @@ -59,8 +44,20 @@ def test_explicit_graph(self): x, t = dl() p = fx(x=x) lss = loss(predictions=p, target=t) + # Bind the loss output. + g0.outputs["loss"] = lss # Instantiate an optimizer to perform the `train` action. optimizer = PtActions() - # Invoke "train" action - perform single forward-backard step. - optimizer.train([lss], optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") + + # Make sure user CANNOT pass training graph and tensors_to_optimize. + with pytest.raises(ValueError): + optimizer.train( + tensors_to_optimize=lss, + training_graph=g0, + optimization_params={"max_steps": 1, "lr": 0.0003}, + optimizer="sgd", + ) + + # But user can invoke "train" action using graph only. + optimizer.train(training_graph=g0, optimization_params={"max_steps": 1, "lr": 0.0003}, optimizer="sgd") diff --git a/tests/integration/core/test_integration_neural_graph_nesting.py b/tests/integration/core/test_integration_neural_graph_nesting.py deleted file mode 100644 index e5a7458da311..000000000000 --- a/tests/integration/core/test_integration_neural_graph_nesting.py +++ /dev/null @@ -1,97 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import pytest -import torch - -from nemo.backends.pytorch.actions import PtActions -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import EvaluatorCallback, NeuralGraph, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - - -@pytest.mark.usefixtures("neural_factory") -class TestNeuralGraphNesting: - @pytest.mark.integration - def test_nesting_operation_modes_example(self): - """ - Tests whether one can nest one graph in mode `both` (representing the our `model`) into - `training` and validation (`inference`) graphs. - """ - # Instantiate the necessary neural modules. - dl_training = RealFunctionDataLayer(n=100, batch_size=4) - dl_validation = RealFunctionDataLayer(n=100, batch_size=4) - fx = TaylorNet(dim=4) - loss = MSELoss() - - with NeuralGraph(operation_mode=OperationMode.both) as model: - # Bind the input. - _ = fx(x=model) - # All outputs will be bound by default. - - # Nest model into training graph. - with NeuralGraph(operation_mode=OperationMode.training) as training_graph: - # Take outputs from the training DL. - x, t = dl_training() - # Pass them to the model - p = model(x=x) - # Pass both of them to loss. - lss = loss(predictions=p, target=t) - - # Nest model into validation graph. - with NeuralGraph(operation_mode=OperationMode.inference) as validation_graph: - # Take outputs from the training DL. - x_valid, t_valid = dl_training() - # Pass them to the model - p_valid = model(x=x_valid) - loss_e = loss(predictions=p_valid, target=t_valid) - - # Callbacks to print info to console and Tensorboard. - train_callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') - ) - - def batch_loss_per_batch_callback(tensors, global_vars): - if "batch_loss" not in global_vars.keys(): - global_vars["batch_loss"] = [] - for key, value in tensors.items(): - if key.startswith("loss"): - global_vars["batch_loss"].append(torch.mean(torch.stack(value))) - - def batch_loss_epoch_finished_callback(global_vars): - epoch_loss = torch.max(torch.tensor(global_vars["batch_loss"])) - logging.info("Evaluation Loss: {0}".format(epoch_loss)) - return dict({"Evaluation Loss": epoch_loss}) - - eval_callback = EvaluatorCallback( - eval_tensors=[loss_e], - user_iter_callback=batch_loss_per_batch_callback, - user_epochs_done_callback=batch_loss_epoch_finished_callback, - eval_step=1, - ) - - # Instantiate an optimizer to perform the `train` action. - optimizer = PtActions() - # Invoke "train" action - perform single forward-backard step. - optimizer.train( - [lss], - callbacks=[train_callback, eval_callback], - optimization_params={"max_steps": 2, "lr": 0.0003}, - optimizer="sgd", - ) From 356e0c28457368ab10ec81313b56dbdc9ce7caef Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 17:17:56 -0700 Subject: [PATCH 091/106] LGTM fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index e3b811e8e378..2aa76541d1d5 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -17,7 +17,7 @@ # ============================================================================= from collections.abc import MutableMapping -from typing import Any, Dict, List, Optional, Union +from typing import Dict, List, Optional, Union from nemo.core.neural_types import NeuralType from nemo.utils import logging From cd1e23eee4ab8e6b8e699df5e9bd14a7b645211d Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 17:23:40 -0700 Subject: [PATCH 092/106] bind description updated Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_inputs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index 2aa76541d1d5..a8030b36123b 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -39,8 +39,10 @@ def __init__(self, ntype: NeuralType): # List of StepModulePort tuples to which this input links to (step number, module name, port name). self._consumers = [] - def bind(self, step_module_ports: StepModulePort): + def bind(self, step_module_ports: Union[StepModulePort, List[StepModulePort]]): """ Binds the (step-module-ports) to this "graph input". + Add "consumers" of this graph input (modules attached to this port), + so when one actually will pass the NmTensor, those modules will be connected. Args: step_module_ports: A single StepModulePort OR a list of StepModulePort tuples to be added. From 37ba9233f8be9b8beaaefc698004054cddd418b5 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Thu, 30 Apr 2020 18:14:26 -0700 Subject: [PATCH 093/106] removed app_state from jasper Signed-off-by: Tomasz Kornuta --- .../start_here/graph_composition_integration_tests0_jasper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index a58030f920c4..66153c874926 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -27,10 +27,8 @@ from nemo.collections.asr.helpers import monitor_asr_train_progress from nemo.core import NeuralGraph, OperationMode from nemo.utils import logging -from nemo.utils.app_state import AppState nf = nemo.core.NeuralModuleFactory() -app_state = AppState() logging.info( "This example shows how one can build a Jasper model using the explicit graph." From e28979e74aaf72e45afcedab966f276195015ce9 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 1 May 2020 16:30:36 -0700 Subject: [PATCH 094/106] graph input_ports and output_ports are now immutable, added some unit tests that check that, minor cleanups Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests1.py | 2 +- nemo/core/neural_graph/graph_inputs.py | 19 +++++-- nemo/core/neural_graph/graph_outputs.py | 43 ++++++++++---- nemo/core/neural_graph/neural_graph.py | 10 ++-- .../core/test_integration_neural_graph.py | 4 +- ...inding.py => test_neural_graph_binding.py} | 56 ++++++++++++++++--- 6 files changed, 104 insertions(+), 30 deletions(-) rename tests/unit/core/neural_graph/{test_graph_outputs_binding.py => test_neural_graph_binding.py} (70%) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py index b6addf4a0d90..5d81956b5eb9 100644 --- a/examples/start_here/graph_composition_integration_tests1.py +++ b/examples/start_here/graph_composition_integration_tests1.py @@ -34,7 +34,7 @@ p = m2(x=x) lss = loss(predictions=p, target=t) # Manual bind. - g0.output_ports["output"] = lss + g0.outputs["output"] = lss # Print the summary. logging.info(g0.summary()) diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/core/neural_graph/graph_inputs.py index a8030b36123b..dcee2b3d0caf 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/core/neural_graph/graph_inputs.py @@ -19,6 +19,8 @@ from collections.abc import MutableMapping from typing import Dict, List, Optional, Union +from frozendict import frozendict + from nemo.core.neural_types import NeuralType from nemo.utils import logging from nemo.utils.connection import StepModulePort @@ -112,15 +114,20 @@ def __setitem__(self, key: str, value: Union[NeuralType, GraphInput]): self._inputs[key] = GraphInput(ntype=ntype) def __getitem__(self, key: str) -> GraphInput: - """ Returns bound input. """ + """ + Returns the bound input associated with the given key. + + Args: + key: Name of the bound input. + """ return self._inputs[key] def __delitem__(self, key: str): """ Raises: - NotImplementedError as deletion of a bound input port is not allowed. + TypeError as deletion of a bound input port is not allowed. """ - raise NotImplementedError("Deletion of a bound input port is not allowed") + raise TypeError("Deletion of a bound input port is not allowed") def __iter__(self): """ @@ -140,12 +147,16 @@ def __len__(self) -> int: def definitions(self) -> Dict[str, NeuralType]: """ Property returns definitions of the input ports by extracting them on the fly from list. + + ..info: + This property actually returns a FrozenDict containing port definitions to indicate that + port definitions SHOULD not be used during the actual binding. Returns: Dictionary of neural types associated with bound inputs. """ # Extract port definitions (Neural Types) from the inputs list. - return {k: v.ntype for k, v in self._inputs.items()} + return frozendict({k: v.ntype for k, v in self._inputs.items()}) def has_binding(self, step_number: int, port_name: str) -> Optional[str]: """ diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 830c75772966..2c9218ca692b 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -19,6 +19,8 @@ from collections.abc import MutableMapping from typing import Any, Dict, List, Optional +from frozendict import frozendict + from nemo.core.neural_types import NeuralType, NmTensor from nemo.utils import logging from nemo.utils.connection import StepModulePort @@ -83,30 +85,43 @@ def __init__(self, tensors_ref): # In this case tring to overwriting the existing ports with new tensors will be forbidden (Exception). self._manual_outputs = {} - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: NmTensor): """ This method is used to set the manual output - creates a GraphOutput item and adds it to the list. Args: - key: name of the output (port). - value: tensor that will be used to create GraphOutput. + key: The name of the output (port). + value: NmTensor that will be used to create a given GraphOutput. """ # Make sure that user passed a NmTensor. - assert type(value).__name__ == "NmTensor" + if not isinstance(value, NmTensor): + raise TypeError("Port `{}` definition must be must be set using a NmTensor".format(key)) + if key in self._manual_outputs.keys(): raise KeyError("Overwriting of a port `{}` that was previously manually bound is not allowed".format(key)) - # Ok, set output. + + # Ok, set thee "manual" output. self._manual_outputs[key] = GraphOutput(value.ntype, value.producer_step_module_port) - def __getitem__(self, key): - """ Returns GraphOutput - depending whether there are some manual outputs or not. """ + def __getitem__(self, key: str) -> GraphOutput: + """ + Returns the bound output associated with the given key. + Uses default or manual dict depending whether there are some manual outputs or not. + + Args: + key: Name of the bound input. + """ if len(self._manual_outputs) > 0: return self._manual_outputs[key] else: # Use default dict. return self._default_outputs[key] - def __delitem__(self, key): - raise NotImplementedError("Deleting a bound output is not allowed") + def __delitem__(self, key: str): + """ + Raises: + TypeError as deletion of a bound input port is not allowed. + """ + raise TypeError("Deleting a bound output is not allowed") def __iter__(self): """ @@ -160,15 +175,21 @@ def bind(self, tensors_ref: List[NmTensor], port_names: Optional[str] = None): def definitions(self) -> Dict[str, GraphOutput]: """ Property returns definitions of the output ports by extracting them on the fly from the bound outputs. + + ..info: + This property actually returns a FrozenDict containing port definitions to indicate that + port definitions SHOULD not be used during the actual binding. + Returns: Dictionary of neural types associated with bound outputs. """ # Get the right output dictionary. d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs - # Extract port definitions (Neural Types). - return {k: v.ntype for k, v in d.items()} + # Extract port definitions (Neural Types) and return an immutable dictionary - so one won't be able + # to try to modify its content by an accident! + return frozendict({k: v.ntype for k, v in d.items()}) @property def tensors(self) -> Dict[str, NmTensor]: diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph/neural_graph.py index 8ef51dd625bf..c9cc4099f844 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph/neural_graph.py @@ -114,12 +114,12 @@ def __call__(self, **kwargs): raise NeuralPortNameMismatchError(port_name) # "Nest" this graph into an active graph. - results = self._app_state.active_graph.nest(self, kwargs) + results = self._app_state.active_graph.__nest(self, kwargs) # Return output tensors. return results - def nest(self, inner_graph: 'NeuralGraph', inner_graph_args): + def __nest(self, inner_graph: 'NeuralGraph', inner_graph_args): """ Method nests (copies) a graph: modules, steps, topology (tensors). @@ -324,7 +324,7 @@ def input_ports(self) -> Dict[str, NeuralType]: Returns definitions of graph input ports (dict of Neural Types). .. note:: - This method actually returns a dictionary with definitions (like Neural Modules). + This method actually returns an immutable dictionary with port types (like Neural Modules). In order to get access to actual graph inputs please call the inputs() method. Returns: @@ -348,7 +348,7 @@ def output_ports(self) -> Dict[str, NeuralType]: Returns definitions of module output ports (dict of Neural Types). .. note:: - This method actually returns a dictionary with definitions (like Neural Modules). + This method actually returns an immutable dictionary with port types (like Neural Modules). In order to get access to actual graph outpus please call the outputs() method. Returns: @@ -916,7 +916,7 @@ def summary(self) -> str: for output in outputs["mappings"]: desc += " * {}\n".format(output) # Line "decorator". - desc += 120 * '=' + "\n" + desc += 120 * '=' # Return the result. return desc diff --git a/tests/integration/core/test_integration_neural_graph.py b/tests/integration/core/test_integration_neural_graph.py index 8b85e62a0de7..bdbcacd64a2c 100644 --- a/tests/integration/core/test_integration_neural_graph.py +++ b/tests/integration/core/test_integration_neural_graph.py @@ -28,8 +28,8 @@ class TestNeuralGraphTrainAction: @pytest.mark.integration def test_explicit_graph(self): """ - Tests the integration of an `explicit` graph and decoupling of graph creation from its activation. - Additionally checks whether user can pass NG instance to train(). + Tests the integration of an `explicit` graph with actions API. + In particular, checks whether user can pass NG instance to train(). """ # Create modules. dl = RealFunctionDataLayer(n=100, batch_size=4) diff --git a/tests/unit/core/neural_graph/test_graph_outputs_binding.py b/tests/unit/core/neural_graph/test_neural_graph_binding.py similarity index 70% rename from tests/unit/core/neural_graph/test_graph_outputs_binding.py rename to tests/unit/core/neural_graph/test_neural_graph_binding.py index b9e00513f439..eaac1d15de3a 100644 --- a/tests/unit/core/neural_graph/test_graph_outputs_binding.py +++ b/tests/unit/core/neural_graph/test_neural_graph_binding.py @@ -27,7 +27,7 @@ @pytest.mark.usefixtures("neural_factory") class TestGraphOutputs: @pytest.mark.unit - def test_graph_outputs1_binding(self): + def test_graph_outputs_binding1(self): # Create modules. data_source = RealFunctionDataLayer(n=100, batch_size=1) tn = TaylorNet(dim=4) @@ -47,7 +47,7 @@ def test_graph_outputs1_binding(self): bound_outputs.bind([lss]) # Delete not allowed. - with pytest.raises(NotImplementedError): + with pytest.raises(TypeError): del bound_outputs["loss"] assert len(bound_outputs) == 4 @@ -68,7 +68,7 @@ def test_graph_outputs1_binding(self): bound_outputs["my_loss"] = lss # Delete not allowed. - with pytest.raises(NotImplementedError): + with pytest.raises(TypeError): del bound_outputs["my_prediction"] assert len(bound_outputs) == 2 @@ -80,7 +80,7 @@ def test_graph_outputs1_binding(self): _ = defs["x"] @pytest.mark.unit - def test_graph_outputs2_binding(self): + def test_graph_outputs_binding2(self): # Create modules. data_source = RealFunctionDataLayer(n=100, batch_size=1, name="tgo2_ds") tn = TaylorNet(dim=4, name="tgo2_tn") @@ -111,10 +111,52 @@ def test_graph_outputs2_binding(self): assert g1.output_tensors[port] is tensor # Test manual binding. - with g1: - g1.outputs["my_prediction"] = y_pred - g1.outputs["my_loss"] = lss + g1.outputs["my_prediction"] = y_pred + g1.outputs["my_loss"] = lss assert len(g1.outputs) == 2 assert g1.output_tensors["my_prediction"].compare(tn.output_ports["y_pred"]) == NeuralTypeComparisonResult.SAME assert g1.output_tensors["my_loss"].compare(loss.output_ports["loss"]) == NeuralTypeComparisonResult.SAME + + # Finally, make sure that the user cannot "bind" "output_ports"! + with pytest.raises(TypeError): + g1.output_ports["my_prediction"] = y_pred + + + @pytest.mark.unit + def test_graph_inputs_binding1_default(self): + # Create modules. + tn = TaylorNet(dim=4, name="tgi1_tn") + loss = MSELoss(name="tgi1_loss") + + # Test default binding. + with NeuralGraph() as g1: + y_pred = tn(x=g1) + lss = loss(predictions=y_pred, target=g1) + + assert len(g1.inputs) == 2 + assert g1.input_ports["x"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g1.input_ports["target"].compare(loss.input_ports["target"]) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_graph_inputs_binding2_manual(self): + # Create modules. + tn = TaylorNet(dim=4, name="tgi1_tn") + loss = MSELoss(name="tgi1_loss") + + # Test "manual" binding. + with NeuralGraph() as g1: + # Bind the "x" input to tn. + g1.inputs["i"] = tn.input_ports["x"] + y_pred = tn(x=g1.inputs["i"]) + # Bing the "target" input to loss. + g1.inputs["t"] = loss.input_ports["target"] + lss = loss(predictions=y_pred, target=g1.inputs["t"]) + + assert len(g1.inputs) == 2 + assert g1.input_ports["i"].compare(tn.input_ports["x"]) == NeuralTypeComparisonResult.SAME + assert g1.input_ports["t"].compare(loss.input_ports["target"]) == NeuralTypeComparisonResult.SAME + + # Finally, make sure that the user cannot "bind" "input_ports"! + with pytest.raises(TypeError): + g1.input_ports["my_prediction"] = y_pred From 693b744c3b784749669338a5a4ffc3d7218fc334 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 1 May 2020 17:28:22 -0700 Subject: [PATCH 095/106] removed a line - formatting fix :] Signed-off-by: Tomasz Kornuta --- tests/unit/core/neural_graph/test_neural_graph_binding.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/core/neural_graph/test_neural_graph_binding.py b/tests/unit/core/neural_graph/test_neural_graph_binding.py index eaac1d15de3a..9db980a7427f 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_binding.py +++ b/tests/unit/core/neural_graph/test_neural_graph_binding.py @@ -122,7 +122,6 @@ def test_graph_outputs_binding2(self): with pytest.raises(TypeError): g1.output_ports["my_prediction"] = y_pred - @pytest.mark.unit def test_graph_inputs_binding1_default(self): # Create modules. From 413c0fa4820bba064c0c28d5ac663808fb7818ea Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 1 May 2020 17:54:39 -0700 Subject: [PATCH 096/106] name fix in test Signed-off-by: Tomasz Kornuta --- tests/unit/core/neural_graph/test_neural_graph_binding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/core/neural_graph/test_neural_graph_binding.py b/tests/unit/core/neural_graph/test_neural_graph_binding.py index 9db980a7427f..0918e780a1c9 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_binding.py +++ b/tests/unit/core/neural_graph/test_neural_graph_binding.py @@ -140,8 +140,8 @@ def test_graph_inputs_binding1_default(self): @pytest.mark.unit def test_graph_inputs_binding2_manual(self): # Create modules. - tn = TaylorNet(dim=4, name="tgi1_tn") - loss = MSELoss(name="tgi1_loss") + tn = TaylorNet(dim=4, name="tgi2_tn") + loss = MSELoss(name="tgi2_loss") # Test "manual" binding. with NeuralGraph() as g1: From ccc1b21d0322a2d93c36d54eb19342a66ddaffec Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 1 May 2020 18:11:44 -0700 Subject: [PATCH 097/106] Minor touches here and there Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 10 ++++++---- nemo/core/neural_modules.py | 2 +- .../core/neural_graph/test_neural_graph_binding.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 2c9218ca692b..174a769f53e5 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -187,8 +187,8 @@ def definitions(self) -> Dict[str, GraphOutput]: # Get the right output dictionary. d = self._manual_outputs if len(self._manual_outputs) > 0 else self._default_outputs - # Extract port definitions (Neural Types) and return an immutable dictionary - so one won't be able - # to try to modify its content by an accident! + # Extract port definitions (Neural Types) and return an immutable dictionary, + # so the user won't be able to modify its content by an accident! return frozendict({k: v.ntype for k, v in d.items()}) @property @@ -204,6 +204,7 @@ def tensors(self) -> Dict[str, NmTensor]: output_tensors = {} # Get tensors by acessing the producer-ports. + # At that point all keys (k) are unigue - we made sure of that during binding/__setitem__. for k, v in d.items(): producer_step = v.producer_step_module_port.step_number producer_port_name = v.producer_step_module_port.port_name @@ -211,8 +212,9 @@ def tensors(self) -> Dict[str, NmTensor]: tensor = self._tensors_ref[producer_step][producer_port_name] # Add it to the dictionary. output_tensors[k] = tensor - # Return the result. - return output_tensors + # Return the result as an immutable dictionary, + # so the user won't be able to modify its content by an accident! + return frozendict(output_tensors) @property def tensor_list(self) -> List[NmTensor]: diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 993861400a1c..44d8ef4a79f4 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -535,7 +535,7 @@ def operation_mode(self): return self._operation_mode @operation_mode.setter - def operation_mode(self, operation_mode): + def operation_mode(self, operation_mode: OperationMode): """ Sets the operation mode. """ self._operation_mode = operation_mode diff --git a/tests/unit/core/neural_graph/test_neural_graph_binding.py b/tests/unit/core/neural_graph/test_neural_graph_binding.py index 0918e780a1c9..c109e33c7369 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_binding.py +++ b/tests/unit/core/neural_graph/test_neural_graph_binding.py @@ -107,7 +107,7 @@ def test_graph_outputs_binding2(self): assert g1.output_ports[port].compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME # Compare definitions - from output_tensors. assert g1.output_tensors[port].compare(module.output_ports[port]) == NeuralTypeComparisonResult.SAME - # Make sure that tensor was bound, i.e. iput refers to the same object instance! + # Make sure that tensor was bound, i.e. input refers to the same object instance! assert g1.output_tensors[port] is tensor # Test manual binding. From a191dbe75939d99c4cc7ed489712f1a5c7700446 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Fri, 1 May 2020 19:31:57 -0700 Subject: [PATCH 098/106] Unique names generates from names of classes Signed-off-by: Tomasz Kornuta --- nemo/utils/object_registry.py | 11 ++++++++--- tests/unit/utils/test_object_registry.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index aed46167eb9e..e97606a92cfa 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -51,7 +51,7 @@ def register(self, new_obj, name: str) -> str: # Check object name. if name is None: # Generate a new, unique name. - unique_name = self.__generate_unique_name() + unique_name = self.__generate_unique_name(new_obj) else: # Check if name is unique. if self.has(name): @@ -78,18 +78,23 @@ def has(self, name: str) -> bool: # Else: return False - def __generate_unique_name(self) -> str: + def __generate_unique_name(self, new_obj) -> str: """ Generates a new unique name by adding postfix (number) to base name. + Args: + new_obj: An object to be registered. + Returns: A generated unique name. """ # Iterate through numbers. postfix = 0 + # Get type name. + base_type_name = (type(new_obj).__name__).lower() while True: # Generate name. - new_name = self._base_type_name + str(postfix) + new_name = base_type_name + str(postfix) # Check uniqueneess. if not self.has(new_name): # Ok, got a unique name! diff --git a/tests/unit/utils/test_object_registry.py b/tests/unit/utils/test_object_registry.py index 6bdb756d6e50..00cc7f964bff 100644 --- a/tests/unit/utils/test_object_registry.py +++ b/tests/unit/utils/test_object_registry.py @@ -46,7 +46,7 @@ def __init__(self, name=None): # Test unique names generation. c3 = MockupObjectClass() c4 = MockupObjectClass() - assert c4.name == "object1" + assert c4.name == "mockupobjectclass1" # Check objects. assert len(registry) == 4 From 68bd7e22edf53ac05ec24d08a27493dc6a3b5ce0 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 4 May 2020 11:06:47 -0700 Subject: [PATCH 099/106] moved graph managed to utils, removed it from init Signed-off-by: Tomasz Kornuta --- .../graph_composition_integration_tests0_jasper.py | 2 +- nemo/core/neural_graph/__init__.py | 1 - nemo/utils/app_state.py | 12 +++++++----- .../neural_graph => utils}/neural_graph_manager.py | 12 +++++++----- 4 files changed, 15 insertions(+), 12 deletions(-) rename nemo/{core/neural_graph => utils}/neural_graph_manager.py (83%) diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py index 66153c874926..695f3669cd1e 100644 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ b/examples/start_here/graph_composition_integration_tests0_jasper.py @@ -77,7 +77,7 @@ # Serialize graph serialized_jasper = Jasper.serialize() -# print("Serialized:\n", serialized_jasper) +print("Serialized:\n", serialized_jasper) # Delete everything - aside of jasper encoder, just as a test to show that reusing work! ;) del Jasper diff --git a/nemo/core/neural_graph/__init__.py b/nemo/core/neural_graph/__init__.py index 7575f45a6ad2..d2f82c09ad36 100644 --- a/nemo/core/neural_graph/__init__.py +++ b/nemo/core/neural_graph/__init__.py @@ -17,4 +17,3 @@ # ============================================================================= from nemo.core.neural_graph.neural_graph import * -from nemo.core.neural_graph.neural_graph_manager import NeuralGraphManager diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index d47f4fa1a5dd..ace5dd613b7f 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -15,11 +15,13 @@ # limitations under the License. # ============================================================================= -# Sadly have to import the whole "nemo" module to avoid circular dependencies. -# Moreover, at that point nemo module doesn't contain core, so during "python module registration" -# nothing from nemo.core, including types (so we cannot use them for "python 3 type hints"). +# Sadly have to import the whole "nemo" python module to avoid circular dependencies. +# Moreover, at that point nemo module doesn't contain "core", so during "python module registration" +# nothing from nemo.core, including e.g. types (so we cannot use them for "python 3 type hints"). import nemo from nemo.utils.metaclasses import Singleton +from nemo.utils.neural_graph_manager import NeuralGraphManager +from nemo.utils.object_registry import ObjectRegistry class AppState(metaclass=Singleton): @@ -42,9 +44,9 @@ def __init__(self, device=None): else: self._device = device # Create module registry. - self._module_registry = nemo.utils.object_registry.ObjectRegistry("module") + self._module_registry = ObjectRegistry("module") # Create graph manager (registry with some additional functionality). - self._neural_graph_manager = nemo.core.NeuralGraphManager() + self._neural_graph_manager = NeuralGraphManager() @property def modules(self): diff --git a/nemo/core/neural_graph/neural_graph_manager.py b/nemo/utils/neural_graph_manager.py similarity index 83% rename from nemo/core/neural_graph/neural_graph_manager.py rename to nemo/utils/neural_graph_manager.py index 3c17216db78d..b6043e7bfc05 100644 --- a/nemo/core/neural_graph/neural_graph_manager.py +++ b/nemo/utils/neural_graph_manager.py @@ -16,8 +16,10 @@ # limitations under the License. # ============================================================================= -from nemo.core.neural_factory import OperationMode -from nemo.core.neural_graph.neural_graph import NeuralGraph +# Sadly have to import the whole "nemo" python module to avoid circular dependencies. +# Moreover, at that point nemo module doesn't contain "core", so during "python module registration" +# nothing from nemo.core, including e.g. types (so we cannot use them for "python 3 type hints"). +import nemo from nemo.utils.object_registry import ObjectRegistry @@ -51,7 +53,7 @@ def summary(self) -> str: return desc @property - def active_graph(self) -> NeuralGraph: + def active_graph(self) -> "NeuralGraph": """ Property returns the active graph. If there is no active graph, creates a new one. @@ -61,7 +63,7 @@ def active_graph(self) -> NeuralGraph: # Create a new graph - training is the default. if self._active_graph is None: # Create a new "default" graph. Default mode: both. - new_graph = NeuralGraph(operation_mode=OperationMode.both) + new_graph = nemo.core.NeuralGraph(operation_mode=core.OperationMode.both) new_graph._name = self.register(new_graph, None) # Set the newly created graph as active. self._active_graph = new_graph @@ -70,7 +72,7 @@ def active_graph(self) -> NeuralGraph: return self._active_graph @active_graph.setter - def active_graph(self, graph: NeuralGraph): + def active_graph(self, graph: "NeuralGraph"): """ Property sets the active graph. From bc216db8d0a3a1fe4c72bd24646754ef01011b12 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 4 May 2020 11:18:35 -0700 Subject: [PATCH 100/106] Moved neural_graph_manager to utils that enable to remove it (along with ObjectRegistry) from __init__ files Signed-off-by: Tomasz Kornuta --- nemo/utils/__init__.py | 1 - nemo/utils/neural_graph_manager.py | 6 ++++-- nemo/utils/object_registry.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index d67c8ee31b77..1ab7e50b43a8 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -24,4 +24,3 @@ from .exp_logging import ExpManager, get_logger from .helpers import * from nemo.utils.app_state import AppState -from nemo.utils.object_registry import ObjectRegistry diff --git a/nemo/utils/neural_graph_manager.py b/nemo/utils/neural_graph_manager.py index b6043e7bfc05..8eaa5e025b20 100644 --- a/nemo/utils/neural_graph_manager.py +++ b/nemo/utils/neural_graph_manager.py @@ -19,7 +19,6 @@ # Sadly have to import the whole "nemo" python module to avoid circular dependencies. # Moreover, at that point nemo module doesn't contain "core", so during "python module registration" # nothing from nemo.core, including e.g. types (so we cannot use them for "python 3 type hints"). -import nemo from nemo.utils.object_registry import ObjectRegistry @@ -62,8 +61,11 @@ def active_graph(self) -> "NeuralGraph": """ # Create a new graph - training is the default. if self._active_graph is None: + # Import core here (to avoid circular dependency between core-utils). + from nemo.core import NeuralGraph, OperationMode + # Create a new "default" graph. Default mode: both. - new_graph = nemo.core.NeuralGraph(operation_mode=core.OperationMode.both) + new_graph = NeuralGraph(operation_mode=OperationMode.both) new_graph._name = self.register(new_graph, None) # Set the newly created graph as active. self._active_graph = new_graph diff --git a/nemo/utils/object_registry.py b/nemo/utils/object_registry.py index e97606a92cfa..8e861e529944 100644 --- a/nemo/utils/object_registry.py +++ b/nemo/utils/object_registry.py @@ -55,7 +55,7 @@ def register(self, new_obj, name: str) -> str: else: # Check if name is unique. if self.has(name): - raise NameError("A {} with name `{}` already exists!".format(self._base_type_name, name)) + raise NameError("A {} with name `{}` already exists!".format(name, name)) # Ok, it is unique. unique_name = name From c54156c6dfbcfb9c03ffbaac46c489eac46645af Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 4 May 2020 15:49:18 -0700 Subject: [PATCH 101/106] removed graph examples Signed-off-by: Tomasz Kornuta --- ...h_composition_integration_tests0_jasper.py | 122 ------------------ .../graph_composition_integration_tests1.py | 48 ------- .../graph_composition_integration_tests2.py | 50 ------- .../graph_composition_integration_tests2_0.py | 52 -------- .../graph_composition_integration_tests2_1.py | 82 ------------ .../graph_composition_integration_tests3.py | 55 -------- .../graph_composition_integration_tests3_1.py | 78 ----------- .../graph_composition_integration_tests4.py | 101 --------------- 8 files changed, 588 deletions(-) delete mode 100644 examples/start_here/graph_composition_integration_tests0_jasper.py delete mode 100644 examples/start_here/graph_composition_integration_tests1.py delete mode 100644 examples/start_here/graph_composition_integration_tests2.py delete mode 100644 examples/start_here/graph_composition_integration_tests2_0.py delete mode 100644 examples/start_here/graph_composition_integration_tests2_1.py delete mode 100644 examples/start_here/graph_composition_integration_tests3.py delete mode 100644 examples/start_here/graph_composition_integration_tests3_1.py delete mode 100644 examples/start_here/graph_composition_integration_tests4.py diff --git a/examples/start_here/graph_composition_integration_tests0_jasper.py b/examples/start_here/graph_composition_integration_tests0_jasper.py deleted file mode 100644 index 695f3669cd1e..000000000000 --- a/examples/start_here/graph_composition_integration_tests0_jasper.py +++ /dev/null @@ -1,122 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from functools import partial -from os.path import expanduser - -from ruamel.yaml import YAML - -import nemo -import nemo.collections.asr as nemo_asr -from nemo.collections.asr.helpers import monitor_asr_train_progress -from nemo.core import NeuralGraph, OperationMode -from nemo.utils import logging - -nf = nemo.core.NeuralModuleFactory() - -logging.info( - "This example shows how one can build a Jasper model using the explicit graph." - F" This approach works for applications containing a single graph." -) - -# Set paths to "manifests" and model configuration files. -train_manifest = "~/TestData/an4_dataset/an4_train.json" -val_manifest = "~/TestData/an4_dataset/an4_val.json" -model_config_file = "~/workspace/nemo/examples/asr/configs/jasper_an4.yaml" - -yaml = YAML(typ="safe") -with open(expanduser(model_config_file)) as f: - config = yaml.load(f) -# Get vocabulary. -vocab = config['labels'] - -# Create neural modules. -data_layer = nemo_asr.AudioToTextDataLayer.deserialize( - config["AudioToTextDataLayer_train"], overwrite_params={"manifest_filepath": train_manifest, "batch_size": 16}, -) - -data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor.deserialize(config["AudioToMelSpectrogramPreprocessor"]) - -jasper_encoder = nemo_asr.JasperEncoder.deserialize(config["JasperEncoder"]) -jasper_decoder = nemo_asr.JasperDecoderForCTC.deserialize( - config["JasperDecoderForCTC"], overwrite_params={"num_classes": len(vocab)} -) -ctc_loss = nemo_asr.CTCLossNM(num_classes=len(vocab)) -greedy_decoder = nemo_asr.GreedyCTCDecoder() - -# Create the Jasper "model". -with NeuralGraph(operation_mode=OperationMode.both) as Jasper: - # Copy one input port definitions - using "user" port names. - Jasper.inputs["input"] = data_preprocessor.input_ports["input_signal"] - # Bind selected inputs - bind other using the default port name. - i_processed_signal, i_processed_signal_len = data_preprocessor(input_signal=Jasper.inputs["input"], length=Jasper) - i_encoded, i_encoded_len = jasper_encoder(audio_signal=i_processed_signal, length=i_processed_signal_len) - i_log_probs = jasper_decoder(encoder_output=i_encoded) - # Bind selected outputs - using "user" port names. - Jasper.outputs["log_probs"] = i_log_probs - Jasper.outputs["encoded_len"] = i_encoded_len - -# Print the summary. -logging.info(Jasper.summary()) - -# Serialize graph -serialized_jasper = Jasper.serialize() -print("Serialized:\n", serialized_jasper) - -# Delete everything - aside of jasper encoder, just as a test to show that reusing work! ;) -del Jasper -del data_preprocessor -# del jasper_encoder # -del jasper_decoder - -# Deserialize graph - copy of the JASPER "model". -jasper_copy = NeuralGraph.deserialize(serialized_jasper, reuse_existing_modules=True, name="jasper_copy") -serialized_jasper_copy = jasper_copy.serialize() -# print("Deserialized:\n", serialized_jasper_copy) -assert serialized_jasper == serialized_jasper_copy - -# Create the "training" graph. -with NeuralGraph(name="training") as training_graph: - # Create the "implicit" training graph. - o_audio_signal, o_audio_signal_len, o_transcript, o_transcript_len = data_layer() - # Use Jasper module as any other neural module. - o_log_probs, o_encoded_len = jasper_copy(input=o_audio_signal, length=o_audio_signal_len) - o_predictions = greedy_decoder(log_probs=o_log_probs) - o_loss = ctc_loss( - log_probs=o_log_probs, targets=o_transcript, input_length=o_encoded_len, target_length=o_transcript_len - ) - # Set graph output. - training_graph.outputs["o_loss"] = o_loss - # training_graph.outputs["o_predictions"] = o_predictions # DOESN'T WORK?!? - -# Print the summary. -logging.info(training_graph.summary()) - -tensors_to_evaluate = [o_loss, o_predictions, o_transcript, o_transcript_len] -train_callback = nemo.core.SimpleLossLoggerCallback( - tensors=tensors_to_evaluate, print_func=partial(monitor_asr_train_progress, labels=vocab) -) -# import pdb;pdb.set_trace() -nf.train( - # tensors_to_optimize=[o_loss, o_predictions], # DOESN'T WORK?!? - training_graph=training_graph, - optimizer="novograd", - callbacks=[train_callback], - optimization_params={"num_epochs": 50, "lr": 0.01}, -) diff --git a/examples/start_here/graph_composition_integration_tests1.py b/examples/start_here/graph_composition_integration_tests1.py deleted file mode 100644 index 5d81956b5eb9..000000000000 --- a/examples/start_here/graph_composition_integration_tests1.py +++ /dev/null @@ -1,48 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32) -m2 = TaylorNet(dim=4) -loss = MSELoss() - -logging.info("This example shows how one can build an `explicit` graph.") - -with NeuralGraph(operation_mode=OperationMode.training) as g0: - x, t = dl() - p = m2(x=x) - lss = loss(predictions=p, target=t) - # Manual bind. - g0.outputs["output"] = lss - -# Print the summary. -logging.info(g0.summary()) - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2.py b/examples/start_here/graph_composition_integration_tests2.py deleted file mode 100644 index 3724ae002c65..000000000000 --- a/examples/start_here/graph_composition_integration_tests2.py +++ /dev/null @@ -1,50 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") -m2 = TaylorNet(dim=4, name="m2") -loss = MSELoss(name="loss") - -logging.info( - "This example shows how one can nest one graph into another - with binding of output ports." - F" Please note that the nested graph can be used exatly like any other module" - F" By default, all output graph ports are bound, thus `visible` outside." -) - -with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: - x, t = dl() - y = m2(x=x) - -with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g11: - x1, t1, p1 = g1() - lss = loss(predictions=p1, target=t1) - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2_0.py b/examples/start_here/graph_composition_integration_tests2_0.py deleted file mode 100644 index 17be99bcc35d..000000000000 --- a/examples/start_here/graph_composition_integration_tests2_0.py +++ /dev/null @@ -1,52 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") -m1 = TaylorNet(dim=4, name="m1") -loss = MSELoss(name="loss") - -logging.info( - "This example shows how one can nest one graph into another - with manual binding of selected output ports." - F" Please note that the nested graph can be used exatly like any other module." -) - -with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: - xg1, tg1 = dl() - -with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: - xg2, tg2 = g1() - pg2 = m1(x=xg2) - lssg2 = loss(predictions=pg2, target=tg2) - - -# import pdb;pdb.set_trace() - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[lssg2], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lssg2], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests2_1.py b/examples/start_here/graph_composition_integration_tests2_1.py deleted file mode 100644 index 5f163c9a818a..000000000000 --- a/examples/start_here/graph_composition_integration_tests2_1.py +++ /dev/null @@ -1,82 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import AppState, logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") -m1 = TaylorNet(dim=4, name="m1") -m2 = TaylorNet(dim=4, name="m2") -loss = MSELoss(name="loss") - -logging.info( - "This example shows how one can nest one graph into another - with manual binding of selected output ports." - F" Please note that the nested graph can be used exatly like any other module." -) - -with NeuralGraph(operation_mode=OperationMode.training, name="g1") as g1: - x, t = dl() - prediction1 = m1(x=x) - prediction2 = m2(x=x) - # Manually bind the selected outputs. - g1.outputs["ix"] = x - g1.outputs["te"] = t - g1.outputs["prediction"] = prediction2 - -# Serialize graph -serialized_g1 = g1.serialize() -print("Serialized:\n", serialized_g1) - -# Delete everything! -del g1 -del dl -del m1 -del m2 - -print(AppState().modules.summary()) -assert len(AppState().modules) == 1 # only the "loss" module - -# Deserialize graph. -g1_copy = NeuralGraph.deserialize(serialized_g1, reuse_existing_modules=False, name="g1_copy") -serialized_g1_copy = g1_copy.serialize() -print("Deserialized:\n", serialized_g1_copy) - - -with NeuralGraph(operation_mode=OperationMode.training, name="g1.1") as g2: - x1, t1, p1 = g1_copy() - lss = loss(predictions=p1, target=t1) - # Manual output. - g2.outputs["loss"] = lss - - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[g2.output_tensors["loss"]], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train( - [g2.output_tensors["loss"]], - callbacks=[callback], - optimization_params={"num_epochs": 2, "lr": 0.0003}, - optimizer="sgd", -) diff --git a/examples/start_here/graph_composition_integration_tests3.py b/examples/start_here/graph_composition_integration_tests3.py deleted file mode 100644 index 11b3372ae8a3..000000000000 --- a/examples/start_here/graph_composition_integration_tests3.py +++ /dev/null @@ -1,55 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32, name="dl") -fx = TaylorNet(dim=4, name="fx") -loss = MSELoss(name="loss") - -logging.info( - "This example shows how one can nest one graph into another - with binding of the input ports." - F" Please note that the nested graph can be used exatly like any other module" - F" In particular, note that the input port 'x' of the module `m2` is bound in graph 'g2'" - F" and then set to `x` returned by `dl` in the graph `g3`." -) - -with NeuralGraph(operation_mode=OperationMode.training, name="g2") as g2: - # Add module to graph and bind it input port 'x'. - y = fx(x=g2) - -# Build the training graph. -with NeuralGraph(operation_mode=OperationMode.training, name="g3") as g3: - # Add modules to graph. - x, t = dl() - # Incorporate modules from existing graph. - pred = g2(x=x) - lss = loss(predictions=pred, target=t) - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") diff --git a/examples/start_here/graph_composition_integration_tests3_1.py b/examples/start_here/graph_composition_integration_tests3_1.py deleted file mode 100644 index c7f3375942f7..000000000000 --- a/examples/start_here/graph_composition_integration_tests3_1.py +++ /dev/null @@ -1,78 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import DeviceType, NeuralGraph, NeuralModuleFactory, OperationMode, SimpleLossLoggerCallback -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl = RealFunctionDataLayer(n=100, batch_size=32, name="real_function_dl") -fx = TaylorNet(dim=4, name="taylor_net") -loss = MSELoss(name="mse_loss") - -logging.info( - "This example shows how one can nest one graph into another - with binding of the input ports." - F" Please note that the nested graph can be used exatly like any other module" - F" In particular, note that the input port 'x' of the module `m2` is bound in graph 'g2'" - F" and then set to `x` returned by `dl` in the graph `g3`." -) - -# Create "model". -with NeuralGraph(operation_mode=OperationMode.both, name="model") as model: - # Manually bind input port: "input" -> "x" - model.inputs["input"] = fx.input_ports["x"] - # Add module to graph and bind it input port 'x'. - y = fx(x=model.inputs["input"]) - # Manual output bind. - model.outputs["output"] = y - -# Serialize graph -serialized_model = model.serialize() -print("Serialized:\n", serialized_model) - -# Delete everything! -# del model -# del fx - -# Deserialize graph. -model_copy = NeuralGraph.deserialize(serialized_model, reuse_existing_modules=True, name="model_copy") - -serialized_model_copy = model_copy.serialize() -print("Deserialized:\n", serialized_model_copy) - -# Build the training graph. -with NeuralGraph(operation_mode=OperationMode.training, name="training") as training: - # Add modules to graph. - x, t = dl() - # Incorporate modules from the existing "model" graph. - p = model_copy(input=x) - lss = loss(predictions=p, target=t) - -# SimpleLossLoggerCallback will print loss values to console. -callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), -) - -# Invoke "train" action. -nf.train([lss], callbacks=[callback], optimization_params={"num_epochs": 2, "lr": 0.0003}, optimizer="sgd") - -# Serialize graph -# print(model.serialize()) -# print(training.serialize()) diff --git a/examples/start_here/graph_composition_integration_tests4.py b/examples/start_here/graph_composition_integration_tests4.py deleted file mode 100644 index eaf3707a694e..000000000000 --- a/examples/start_here/graph_composition_integration_tests4.py +++ /dev/null @@ -1,101 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import torch - -from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet -from nemo.core import ( - DeviceType, - EvaluatorCallback, - NeuralGraph, - NeuralModuleFactory, - OperationMode, - SimpleLossLoggerCallback, -) -from nemo.utils import logging - -nf = NeuralModuleFactory(placement=DeviceType.CPU) -# Instantiate the necessary neural modules. -dl_training = RealFunctionDataLayer(n=100, batch_size=32) -dl_validation = RealFunctionDataLayer(n=100, batch_size=32) -fx = TaylorNet(dim=4) -loss = MSELoss() - -logging.info( - "This example shows how one can nest one graph (representing the our trained model) into" - F" training and validation graphs." -) - -# Build the training graph. -with NeuralGraph(operation_mode=OperationMode.both, name="trainable_module") as trainable_module: - # Bind the input. - _ = fx(x=trainable_module) - # All outputs will be bound by default. - -# Compose two graphs into final graph. -with NeuralGraph(operation_mode=OperationMode.training, name="training_graph") as training_graph: - # Take outputs from the training DL. - x, t = dl_training() - # Pass them to the trainable module. - p = trainable_module(x=x) - # Pass both of them to loss. - lss = loss(predictions=p, target=t) - -with NeuralGraph(operation_mode=OperationMode.inference, name="validation_graph") as validation_graph: - # Take outputs from the training DL. - x_valid, t_valid = dl_training() - # Pass them to the trainable module. - p_valid = trainable_module(x=x_valid) - loss_e = loss(predictions=p_valid, target=t_valid) - - -# Callbacks to print info to console and Tensorboard. -train_callback = SimpleLossLoggerCallback( - tensors=[lss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}') -) - - -def batch_loss_per_batch_callback(tensors, global_vars): - if "batch_loss" not in global_vars.keys(): - global_vars["batch_loss"] = [] - for key, value in tensors.items(): - if key.startswith("loss"): - global_vars["batch_loss"].append(torch.mean(torch.stack(value))) - - -def batch_loss_epoch_finished_callback(global_vars): - epoch_loss = torch.max(torch.tensor(global_vars["batch_loss"])) - print("Evaluation Loss: {0}".format(epoch_loss)) - return dict({"Evaluation Loss": epoch_loss}) - - -eval_callback = EvaluatorCallback( - eval_tensors=[loss_e], - user_iter_callback=batch_loss_per_batch_callback, - user_epochs_done_callback=batch_loss_epoch_finished_callback, - eval_step=100, -) - -# Invoke "train" action. -nf.train( - [lss], - callbacks=[train_callback, eval_callback], - optimization_params={"num_epochs": 3, "lr": 0.0003}, - optimizer="sgd", -) From cf72cf8fe2550b9ed3f15ddf0ec741048e3584b8 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 4 May 2020 15:57:20 -0700 Subject: [PATCH 102/106] Updated docstring in GraphInputs Signed-off-by: Tomasz Kornuta --- nemo/core/neural_graph/graph_outputs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/core/neural_graph/graph_outputs.py index 174a769f53e5..cae19d72c9e7 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/core/neural_graph/graph_outputs.py @@ -58,11 +58,11 @@ class GraphOutputs(MutableMapping): ''' A specialized dictionary that contains bound outputs of a Neural Graph. In fact stores two lists of "outputs": - - "default" outputs with default keys taken from outputs of modules (might result in - overwriting some keys), and - - "manual" used for specifying the subset of outputs, each with a new/different key - When accessing the outputs, it returns the "manual" outputs. If "manual" outputs are not defined, - will return/work on "default" outputs. + - "default" outputs with default keys taken from outputs of modules, and + - "manual" used for specifying the subset of outputs. + When accessing the outputs, it returns the one of those two lists following the rule: + return "manual" outputs if they were definde (at least one manual output defined by the user), + otherwise return the "default" outputs. ''' def __init__(self, tensors_ref): @@ -78,7 +78,7 @@ def __init__(self, tensors_ref): # This dictionary stores the output tensors collected during the "default" tensor recording. # As they are using the default port names, the second/next tensor published on the same port - # will overwrite the old one (Warning). + # will generate a new unique name following the (step_number.module.port_name) pattern. self._default_outputs = {} # This dictionary stores list of output tensors of module "manually" indicated by the user. @@ -168,7 +168,7 @@ def bind(self, tensors_ref: List[NmTensor], port_names: Optional[str] = None): tensor.name, tensor.producer_step_number, tensor.producer_name, name ) ) - # Still, "overwrite" it. + # Store the output. self._default_outputs[name] = GraphOutput(tensor.ntype, tensor.producer_step_module_port) @property From 327cc6d53cf7ff772267dedca07fe50e773688fd Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 4 May 2020 16:57:04 -0700 Subject: [PATCH 103/106] Moved all NeuralGraph helper classes to nemo.utils.neural_graph. AppState remained in the nemo.utils Signed-off-by: Tomasz Kornuta --- nemo/core/{neural_graph => }/neural_graph.py | 6 +++--- nemo/core/neural_graph/__init__.py | 19 ------------------- nemo/core/neural_modules.py | 2 +- nemo/core/neural_types/neural_type.py | 2 +- nemo/utils/__init__.py | 1 - nemo/utils/app_state.py | 4 ++-- nemo/utils/{ => neural_graph}/connection.py | 0 .../neural_graph/graph_inputs.py | 15 +++++++-------- .../neural_graph/graph_outputs.py | 17 ++++++++--------- .../neural_graph_manager.py | 2 +- .../{ => neural_graph}/object_registry.py | 0 .../neural_graph/test_neural_graph_binding.py | 2 +- tests/unit/utils/test_object_registry.py | 2 +- 13 files changed, 25 insertions(+), 47 deletions(-) rename nemo/core/{neural_graph => }/neural_graph.py (99%) delete mode 100644 nemo/core/neural_graph/__init__.py rename nemo/utils/{ => neural_graph}/connection.py (100%) rename nemo/{core => utils}/neural_graph/graph_inputs.py (95%) rename nemo/{core => utils}/neural_graph/graph_outputs.py (96%) rename nemo/utils/{ => neural_graph}/neural_graph_manager.py (97%) rename nemo/utils/{ => neural_graph}/object_registry.py (100%) diff --git a/nemo/core/neural_graph/neural_graph.py b/nemo/core/neural_graph.py similarity index 99% rename from nemo/core/neural_graph/neural_graph.py rename to nemo/core/neural_graph.py index c9cc4099f844..98bef20a05eb 100644 --- a/nemo/core/neural_graph/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -27,14 +27,14 @@ from ruamel.yaml import YAML from nemo.core import OperationMode -from nemo.core.neural_graph.graph_inputs import GraphInputs -from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_interface import NeuralInterface from nemo.core.neural_modules import NeuralModule from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.connection import Connection, StepModulePort +from nemo.utils.neural_graph.connection import Connection, StepModulePort +from nemo.utils.neural_graph.graph_inputs import GraphInputs +from nemo.utils.neural_graph.graph_outputs import GraphOutputs YAML = YAML(typ='safe') diff --git a/nemo/core/neural_graph/__init__.py b/nemo/core/neural_graph/__init__.py deleted file mode 100644 index d2f82c09ad36..000000000000 --- a/nemo/core/neural_graph/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -# ============================================================================= -# Copyright (c) 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from nemo.core.neural_graph.neural_graph import * diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 44d8ef4a79f4..24abfb46264d 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -33,8 +33,8 @@ from nemo.core.neural_types import NeuralPortNameMismatchError, NeuralType, NmTensor from nemo.package_info import __version__ as nemo_version from nemo.utils import logging -from nemo.utils.connection import StepModulePort from nemo.utils.decorators.deprecated import deprecated +from nemo.utils.neural_graph.connection import StepModulePort YAML = YAML(typ='safe') diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 3d72b1ee5d9f..d503f8b78cf1 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -29,7 +29,7 @@ from nemo.core.neural_types.comparison import NeuralTypeComparisonResult from nemo.core.neural_types.elements import * from nemo.utils.app_state import AppState -from nemo.utils.connection import Connection, StepModulePort +from nemo.utils.neural_graph.connection import Connection, StepModulePort class NeuralType(object): diff --git a/nemo/utils/__init__.py b/nemo/utils/__init__.py index 1ab7e50b43a8..b9058a854c3c 100644 --- a/nemo/utils/__init__.py +++ b/nemo/utils/__init__.py @@ -23,4 +23,3 @@ from .argparse import NemoArgParser from .exp_logging import ExpManager, get_logger from .helpers import * -from nemo.utils.app_state import AppState diff --git a/nemo/utils/app_state.py b/nemo/utils/app_state.py index ace5dd613b7f..d77daa133adf 100644 --- a/nemo/utils/app_state.py +++ b/nemo/utils/app_state.py @@ -20,8 +20,8 @@ # nothing from nemo.core, including e.g. types (so we cannot use them for "python 3 type hints"). import nemo from nemo.utils.metaclasses import Singleton -from nemo.utils.neural_graph_manager import NeuralGraphManager -from nemo.utils.object_registry import ObjectRegistry +from nemo.utils.neural_graph.neural_graph_manager import NeuralGraphManager +from nemo.utils.neural_graph.object_registry import ObjectRegistry class AppState(metaclass=Singleton): diff --git a/nemo/utils/connection.py b/nemo/utils/neural_graph/connection.py similarity index 100% rename from nemo/utils/connection.py rename to nemo/utils/neural_graph/connection.py diff --git a/nemo/core/neural_graph/graph_inputs.py b/nemo/utils/neural_graph/graph_inputs.py similarity index 95% rename from nemo/core/neural_graph/graph_inputs.py rename to nemo/utils/neural_graph/graph_inputs.py index dcee2b3d0caf..aa18bde8fc29 100644 --- a/nemo/core/neural_graph/graph_inputs.py +++ b/nemo/utils/neural_graph/graph_inputs.py @@ -21,15 +21,14 @@ from frozendict import frozendict -from nemo.core.neural_types import NeuralType from nemo.utils import logging -from nemo.utils.connection import StepModulePort +from nemo.utils.neural_graph.connection import StepModulePort class GraphInput(object): """ A helper class represenging a single bound input. """ - def __init__(self, ntype: NeuralType): + def __init__(self, ntype: "NeuralType"): """ Initializes object. @@ -57,7 +56,7 @@ def bind(self, step_module_ports: Union[StepModulePort, List[StepModulePort]]): self._consumers.append(smp) @property - def ntype(self) -> NeuralType: + def ntype(self) -> "NeuralType": """ Returns: NeuralType of a given input. @@ -84,7 +83,7 @@ def __init__(self): """ self._inputs = {} - def __setitem__(self, key: str, value: Union[NeuralType, GraphInput]): + def __setitem__(self, key: str, value: Union["NeuralType", GraphInput]): """ This method is used to "create" a bound input, i.e. copy definition from indicated module input port. @@ -96,8 +95,8 @@ def __setitem__(self, key: str, value: Union[NeuralType, GraphInput]): KeyError: Definition of a previously bound port is not allowed. TypeError: Port definition must be must be a NeuralType or GraphInput type. """ - # Make sure that a proper NeuralType definition was passed here. - if isinstance(value, NeuralType): + # Make sure that a proper object was passed here. + if type(value).__name__ == "NeuralType": ntype = value elif isinstance(value, GraphInput): ntype = value.ntype @@ -144,7 +143,7 @@ def __len__(self) -> int: return len(self._inputs) @property - def definitions(self) -> Dict[str, NeuralType]: + def definitions(self) -> Dict[str, "NeuralType"]: """ Property returns definitions of the input ports by extracting them on the fly from list. diff --git a/nemo/core/neural_graph/graph_outputs.py b/nemo/utils/neural_graph/graph_outputs.py similarity index 96% rename from nemo/core/neural_graph/graph_outputs.py rename to nemo/utils/neural_graph/graph_outputs.py index cae19d72c9e7..877e5983e96d 100644 --- a/nemo/core/neural_graph/graph_outputs.py +++ b/nemo/utils/neural_graph/graph_outputs.py @@ -21,15 +21,14 @@ from frozendict import frozendict -from nemo.core.neural_types import NeuralType, NmTensor from nemo.utils import logging -from nemo.utils.connection import StepModulePort +from nemo.utils.neural_graph.connection import StepModulePort class GraphOutput(object): """ A helper class represenging a single bound output. """ - def __init__(self, ntype: NeuralType, producer_step_module_port: StepModulePort): + def __init__(self, ntype: "NeuralType", producer_step_module_port: StepModulePort): """ Initializes object. @@ -41,7 +40,7 @@ def __init__(self, ntype: NeuralType, producer_step_module_port: StepModulePort) self._producer_step_module_port = producer_step_module_port @property - def ntype(self) -> NeuralType: + def ntype(self) -> "NeuralType": """ Returns: NeuralType of a given output. @@ -85,7 +84,7 @@ def __init__(self, tensors_ref): # In this case tring to overwriting the existing ports with new tensors will be forbidden (Exception). self._manual_outputs = {} - def __setitem__(self, key: str, value: NmTensor): + def __setitem__(self, key: str, value: "NmTensor"): """ This method is used to set the manual output - creates a GraphOutput item and adds it to the list. @@ -94,7 +93,7 @@ def __setitem__(self, key: str, value: NmTensor): value: NmTensor that will be used to create a given GraphOutput. """ # Make sure that user passed a NmTensor. - if not isinstance(value, NmTensor): + if type(value).__name__ != "NmTensor": raise TypeError("Port `{}` definition must be must be set using a NmTensor".format(key)) if key in self._manual_outputs.keys(): @@ -143,7 +142,7 @@ def __len__(self) -> int: else: # Use default dict. return len(self._default_outputs) - def bind(self, tensors_ref: List[NmTensor], port_names: Optional[str] = None): + def bind(self, tensors_ref: List["NmTensor"], port_names: Optional[str] = None): """ Binds the "default" outputs. @@ -192,7 +191,7 @@ def definitions(self) -> Dict[str, GraphOutput]: return frozendict({k: v.ntype for k, v in d.items()}) @property - def tensors(self) -> Dict[str, NmTensor]: + def tensors(self) -> Dict[str, "NmTensor"]: """ Property returns output tensors by extracting them on the fly from the bound outputs. @@ -217,7 +216,7 @@ def tensors(self) -> Dict[str, NmTensor]: return frozendict(output_tensors) @property - def tensor_list(self) -> List[NmTensor]: + def tensor_list(self) -> List["NmTensor"]: """ Property returns output tensors by extracting them on the fly from the bound outputs. diff --git a/nemo/utils/neural_graph_manager.py b/nemo/utils/neural_graph/neural_graph_manager.py similarity index 97% rename from nemo/utils/neural_graph_manager.py rename to nemo/utils/neural_graph/neural_graph_manager.py index 8eaa5e025b20..b016b57dc3b8 100644 --- a/nemo/utils/neural_graph_manager.py +++ b/nemo/utils/neural_graph/neural_graph_manager.py @@ -19,7 +19,7 @@ # Sadly have to import the whole "nemo" python module to avoid circular dependencies. # Moreover, at that point nemo module doesn't contain "core", so during "python module registration" # nothing from nemo.core, including e.g. types (so we cannot use them for "python 3 type hints"). -from nemo.utils.object_registry import ObjectRegistry +from nemo.utils.neural_graph.object_registry import ObjectRegistry class NeuralGraphManager(ObjectRegistry): diff --git a/nemo/utils/object_registry.py b/nemo/utils/neural_graph/object_registry.py similarity index 100% rename from nemo/utils/object_registry.py rename to nemo/utils/neural_graph/object_registry.py diff --git a/tests/unit/core/neural_graph/test_neural_graph_binding.py b/tests/unit/core/neural_graph/test_neural_graph_binding.py index c109e33c7369..6ca5a10bf5d7 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_binding.py +++ b/tests/unit/core/neural_graph/test_neural_graph_binding.py @@ -20,8 +20,8 @@ from nemo.backends.pytorch.tutorials import MSELoss, RealFunctionDataLayer, TaylorNet from nemo.core import NeuralGraph, OperationMode -from nemo.core.neural_graph.graph_outputs import GraphOutputs from nemo.core.neural_types import NeuralTypeComparisonResult +from nemo.utils.neural_graph.graph_outputs import GraphOutputs @pytest.mark.usefixtures("neural_factory") diff --git a/tests/unit/utils/test_object_registry.py b/tests/unit/utils/test_object_registry.py index 00cc7f964bff..0132a593bf6f 100644 --- a/tests/unit/utils/test_object_registry.py +++ b/tests/unit/utils/test_object_registry.py @@ -18,7 +18,7 @@ import pytest -from nemo.utils.object_registry import ObjectRegistry +from nemo.utils.neural_graph.object_registry import ObjectRegistry class TestAppState: From 831d269c1fb7a7dcd185847c22523aa510a782c1 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 5 May 2020 11:19:25 -0700 Subject: [PATCH 104/106] inference -> evaluation, as requested Signed-off-by: Tomasz Kornuta --- nemo/backends/pytorch/actions.py | 6 +++--- nemo/core/neural_graph.py | 10 +++++----- .../core/neural_graph/test_neural_graph_nesting.py | 12 ++++++------ .../neural_graph/test_neural_graph_serialization.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index b6c71b37a03a..6407e10a6c52 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -423,7 +423,7 @@ def __nm_graph_forward_pass( # if module.is_trainable(): if isinstance(pmodule, nn.Module): pmodule.train() - elif mode == OperationMode.inference: + elif mode == OperationMode.evaluation: # if module.is_trainable(): if isinstance(pmodule, nn.Module): pmodule.eval() @@ -584,7 +584,7 @@ def _eval(self, tensors_2_evaluate, callback, step, verbose=False): t.unique_name: d for t, d in zip(call_chain[0][2].values(), tensors) if t is not None } self.__nm_graph_forward_pass( - call_chain=call_chain, registered_tensors=registered_e_tensors, mode=OperationMode.inference, + call_chain=call_chain, registered_tensors=registered_e_tensors, mode=OperationMode.evaluation, ) if not is_distributed or self.global_rank == 0: @@ -766,7 +766,7 @@ def _infer( self.__nm_graph_forward_pass( call_chain=call_chain, registered_tensors=registered_e_tensors, - mode=OperationMode.inference, + mode=OperationMode.evaluation, use_cache=use_cache, ) diff --git a/nemo/core/neural_graph.py b/nemo/core/neural_graph.py index 98bef20a05eb..850f2879eefa 100644 --- a/nemo/core/neural_graph.py +++ b/nemo/core/neural_graph.py @@ -95,16 +95,16 @@ def __call__(self, **kwargs): outer_mode = self._app_state.active_graph.operation_mode inner_mode = self.operation_mode - if inner_mode == OperationMode.inference and outer_mode == OperationMode.training: + if inner_mode == OperationMode.evaluation and outer_mode == OperationMode.training: raise TypeError("Cannot nest 'inference' graph into 'training'") - if inner_mode == OperationMode.training and outer_mode == OperationMode.inference: + if inner_mode == OperationMode.training and outer_mode == OperationMode.evaluation: raise TypeError("Cannot nest 'training' graph into 'inference'") if inner_mode == OperationMode.training and outer_mode == OperationMode.both: raise TypeError("Cannot nest 'training' graph into 'both'") - if inner_mode == OperationMode.inference and outer_mode == OperationMode.both: + if inner_mode == OperationMode.evaluation and outer_mode == OperationMode.both: raise TypeError("Cannot nest 'inference' graph into 'both'") # Check inputs: iterate through all inputs passed to the "self". @@ -525,7 +525,7 @@ def __serialize_header(self) -> Dict[str, Any]: # Add operation mode. if self._operation_mode == OperationMode.training: header["operation_mode"] = "training" - elif self._operation_mode == OperationMode.inference: + elif self._operation_mode == OperationMode.evaluation: header["operation_mode"] = "inference" else: header["operation_mode"] = "both" @@ -717,7 +717,7 @@ def __deserialize_header(cls, serialized_header: Dict[str, Any]): if serialized_header["operation_mode"] == "training": operation_mode = OperationMode.training elif serialized_header["operation_mode"] == "inference": - operation_mode = OperationMode.inference + operation_mode = OperationMode.evaluation else: operation_mode = OperationMode.both diff --git a/tests/unit/core/neural_graph/test_neural_graph_nesting.py b/tests/unit/core/neural_graph/test_neural_graph_nesting.py index c5d3d4b6e2bd..09340d5aaee2 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_nesting.py +++ b/tests/unit/core/neural_graph/test_neural_graph_nesting.py @@ -45,9 +45,9 @@ def test_module_nesting1_change_operation_modes(self): _, _ = dl() assert dl.operation_mode == OperationMode.training - with NeuralGraph(operation_mode=OperationMode.inference): + with NeuralGraph(operation_mode=OperationMode.evaluation): _, _ = dl() - assert dl.operation_mode == OperationMode.inference + assert dl.operation_mode == OperationMode.evaluation @pytest.mark.unit def test_graph_nesting2_possible_operation_modes(self): @@ -63,7 +63,7 @@ def test_graph_nesting2_possible_operation_modes(self): with NeuralGraph(operation_mode=OperationMode.training) as training: _, _ = dl() - with NeuralGraph(operation_mode=OperationMode.inference) as inference: + with NeuralGraph(operation_mode=OperationMode.evaluation) as inference: _, _ = dl() # Allowed operations. @@ -72,7 +72,7 @@ def test_graph_nesting2_possible_operation_modes(self): _, _ = both() # Can nest 'both' into 'inference'. - with NeuralGraph(operation_mode=OperationMode.inference): + with NeuralGraph(operation_mode=OperationMode.evaluation): _, _ = both() # Can nest 'training' into 'training'. @@ -80,7 +80,7 @@ def test_graph_nesting2_possible_operation_modes(self): _, _ = training() # Can nest 'inference' into 'inference'. - with NeuralGraph(operation_mode=OperationMode.inference): + with NeuralGraph(operation_mode=OperationMode.evaluation): _, _ = inference() # Can nest 'both' into 'both'. @@ -95,7 +95,7 @@ def test_graph_nesting2_possible_operation_modes(self): # Cannot nest 'training' into 'inference'. with pytest.raises(TypeError): - with NeuralGraph(operation_mode=OperationMode.inference): + with NeuralGraph(operation_mode=OperationMode.evaluation): _, _ = training() # Cannot nest 'training' into 'both'. diff --git a/tests/unit/core/neural_graph/test_neural_graph_serialization.py b/tests/unit/core/neural_graph/test_neural_graph_serialization.py index 76852b9225a8..8956c2b5052b 100644 --- a/tests/unit/core/neural_graph/test_neural_graph_serialization.py +++ b/tests/unit/core/neural_graph/test_neural_graph_serialization.py @@ -81,7 +81,7 @@ def test_graph_serialization_2_simple_graph_output_binding(self): loss = MSELoss(name="tgs2_loss") # Create the graph. - with NeuralGraph(operation_mode=OperationMode.inference) as g1: + with NeuralGraph(operation_mode=OperationMode.evaluation) as g1: x, t = dl() prediction1 = tn(x=x) _ = loss(predictions=prediction1, target=t) From d2615e1d3e44971f0171d01b09f13e1bffc97af3 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 5 May 2020 11:24:15 -0700 Subject: [PATCH 105/106] fix Signed-off-by: Tomasz Kornuta --- nemo/core/neural_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 8aecda3d64d9..4402ded7b927 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -62,7 +62,7 @@ class OperationMode(Enum): """Training or Inference (Evaluation) mode""" training = 0 - inference = 1 + evaluation = 1 both = 2 From da91ec0c60eb456ae1adac718f0a678bba98a3d8 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 5 May 2020 13:01:59 -0700 Subject: [PATCH 106/106] changelog updated Signed-off-by: Tomasz Kornuta --- CHANGELOG.md | 5 +++++ nemo/utils/neural_graph/graph_outputs.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71c9688290e..923ecb78cdd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,11 @@ To release a new version, please update the changelog as followed: ### Contributors +## [0.10.2] - 2020-05-05 + +### Added +- The Neural Graph is a high-level abstract concept empowering the users to build graphs consisting of many, interconnected Neural Modules. A user in his/her application can build any number of graphs, potentially spanning over the same modules. The import/export options combined with the lightweight API make Neural Graphs a perfect tool for rapid prototyping and experimentation. ([PR #413](https://github.com/NVIDIA/NeMo/pull/413)) - @tkornuta + ## [0.10.0] - 2020-04-03 ### Added diff --git a/nemo/utils/neural_graph/graph_outputs.py b/nemo/utils/neural_graph/graph_outputs.py index 877e5983e96d..2210ff316215 100644 --- a/nemo/utils/neural_graph/graph_outputs.py +++ b/nemo/utils/neural_graph/graph_outputs.py @@ -60,7 +60,7 @@ class GraphOutputs(MutableMapping): - "default" outputs with default keys taken from outputs of modules, and - "manual" used for specifying the subset of outputs. When accessing the outputs, it returns the one of those two lists following the rule: - return "manual" outputs if they were definde (at least one manual output defined by the user), + return "manual" outputs if they were define (at least one manual output defined by the user), otherwise return the "default" outputs. '''