From 72094a9b6b7c9e9739880f622b70985759c7e1d1 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 28 Jan 2020 16:18:35 -0800 Subject: [PATCH 01/70] initial draft Signed-off-by: Oleksii Kuchaiev --- nemo/core/neural_types/__init__.py | 21 ++ nemo/core/neural_types/axes.py | 77 +++++++ nemo/core/neural_types/comparison.py | 34 ++++ nemo/core/neural_types/elements.py | 83 ++++++++ nemo/core/neural_types/neural_type.py | 190 ++++++++++++++++++ .../{neural_types.py => old_neural_types.py} | 0 6 files changed, 405 insertions(+) create mode 100644 nemo/core/neural_types/__init__.py create mode 100644 nemo/core/neural_types/axes.py create mode 100644 nemo/core/neural_types/comparison.py create mode 100644 nemo/core/neural_types/elements.py create mode 100644 nemo/core/neural_types/neural_type.py rename nemo/core/{neural_types.py => old_neural_types.py} (100%) diff --git a/nemo/core/neural_types/__init__.py b/nemo/core/neural_types/__init__.py new file mode 100644 index 000000000000..92c9b37c32b6 --- /dev/null +++ b/nemo/core/neural_types/__init__.py @@ -0,0 +1,21 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .elements import * +from .axes import * +from .comparison import * +from .neural_type import * \ No newline at end of file diff --git a/nemo/core/neural_types/axes.py b/nemo/core/neural_types/axes.py new file mode 100644 index 000000000000..5efba5d20ca7 --- /dev/null +++ b/nemo/core/neural_types/axes.py @@ -0,0 +1,77 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['AxisKindAbstract', 'AxisKind', 'AxisType'] + +from enum import Enum +from typing import Optional + + +class AxisKindAbstract(Enum): + """This is an abstract Enum to represents what does varying axis dimension mean. + In practice, you will almost always use AxisKind Enum. This Enum should be inherited by + your OWN Enum if you aren't satisfied with AxisKind. Then your own Enum can be used + instead of AxisKind.""" + + pass + + +class AxisKind(AxisKindAbstract): + """This Enum represents what does varying axis dimension mean. + For example, does this dimension correspond to width, batch, time, etc.""" + + Batch = 0 + Time = 1 + Dimension = 2 + Width = 3 + Height = 4 + + def __str__(self): + return str(self.name).lower() + + @staticmethod + def from_str(label): + """Returns AxisKind instance based on short string representation""" + _label = label.lower().strip() + if _label == "b" or _label == "n" or _label == "batch": + return AxisKind.Batch + elif _label == "t" or _label == "time": + return AxisKind.Time + elif _label == "d" or _label == "c" or _label == "channel": + return AxisKind.Dimension + elif _label == "w" or _label == "width": + return AxisKind.Width + elif _label == "h" or _label == "height": + return AxisKind.Height + else: + raise ValueError(f"Can't create AxisKind from {label}") + + +class AxisType(object): + """This class represents axis semantics and (optionally) it's dimensionality + Args: + kind (AxisKindAbstract): + size (int, optional): + is_list (bool, default=False): + """ + + def __init__(self, kind: AxisKindAbstract, size: Optional[int], is_list=False): + if size is not None and is_list: + raise ValueError("The axis can't be list and have a fixed size") + self.kind = kind + self.size = size + self.is_list = is_list diff --git a/nemo/core/neural_types/comparison.py b/nemo/core/neural_types/comparison.py new file mode 100644 index 000000000000..6cbb9661a0e2 --- /dev/null +++ b/nemo/core/neural_types/comparison.py @@ -0,0 +1,34 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['NeuralTypeComparisonResult'] + +from enum import Enum + + +class NeuralTypeComparisonResult(Enum): + """The result of comparing two neural type objects for compatibility. + When comparing A.compare_to(B):""" + + SAME = 0 + LESS = 1 # A is B + GREATER = 2 # B is A + DIM_INCOMPATIBLE = 3 # Resize connector might fix incompatibility + TRANSPOSE_SAME = 4 # A transpose and/or converting between lists and tensors will make them same + CONTAINER_SIZE_MISMATCH = 5 # A and B contain different number of elements + INCOMPATIBLE = 6 # A and B are incompatible + SAME_TYPE_INCOMPATIBLE_PARAMS = 7 # A and B are of the same type but parametrized differently diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py new file mode 100644 index 000000000000..0b3626556b96 --- /dev/null +++ b/nemo/core/neural_types/elements.py @@ -0,0 +1,83 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['ElementType', 'VoidType'] +import abc +from abc import ABC, abstractmethod +from typing import Tuple, Optional, Dict +from .comparison import NeuralTypeComparisonResult + + +class ElementType(ABC): + """Abstract class defining semantics of the tensor elements. + We are replying on Python for inheritance checking""" + + @abstractmethod + def __str__(cls): + pass + + @property + def type_parameters(self) -> Dict: + """Override this property to parametrize your type""" + return {} + + @property + def fields(self) -> Optional[Tuple]: + return None + + def compare(self, second) -> NeuralTypeComparisonResult: + # First, check general compatibility + result = NeuralTypeComparisonResult.SAME + first_t = type(self) + second_t = type(second) + + if first_t == second_t: + result = NeuralTypeComparisonResult.SAME + elif issubclass(first_t, second_t): + result = NeuralTypeComparisonResult.LESS + elif issubclass(second_t, first_t): + result = NeuralTypeComparisonResult.GREATER + else: + result = NeuralTypeComparisonResult.INCOMPATIBLE + + if result != NeuralTypeComparisonResult.SAME: + return result + else: + # now check that all parameters match + check_params = set(self.type_parameters.keys()) == set(second.type_parameters.keys()) + if check_params is False: + return NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + else: + for k1, v1 in self.type_parameters.items(): + if v1 != second.type_parameters[k1]: + return NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + # check that all fields match + if self.fields == second.fields: + return NeuralTypeComparisonResult.SAME + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + + +class VoidType(ElementType): + """Void-like type which is compatible with everything + """ + + def __str__(self): + return str("void type. compatible with everything") + + def compare(cls, second: abc.ABCMeta) -> NeuralTypeComparisonResult: + return NeuralTypeComparisonResult.SAME diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py new file mode 100644 index 000000000000..cbb216ef80ef --- /dev/null +++ b/nemo/core/neural_types/neural_type.py @@ -0,0 +1,190 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = [ + 'NeuralType', + 'NmTensor', + 'NeuralTypeError', + 'NeuralPortNameMismatchError', + 'NeuralPortNmTensorMismatchError', + 'NeuralPortNmTensorMismatchError', + 'CanNotInferResultNeuralType', +] +import uuid +from typing import Tuple +from .comparison import NeuralTypeComparisonResult +from .axes import AxisType, AxisKind +from .elements import * + + +class NeuralType(object): + """This is the main class which would represent neural type concept. + nmTensors derives from this. It is used to represent *the types* of inputs and outputs.""" + + def __init__(self, elements_type: ElementType, axes: Tuple, optional=False): + self.__check_sanity(axes) + self.elements_type = elements_type + axes_list = [] + for axis in axes: + if isinstance(axis, str): + axes_list.append(AxisType(AxisKind.from_str(axis), None)) + elif isinstance(axis, AxisType): + axes_list.append(axis) + else: + raise ValueError(f"axis type must be either str or AxisType instance") + self.axes_tuple = tuple(axes_list) + self.optional = optional + + def compare(self, second) -> NeuralTypeComparisonResult: + # First, handle dimensionality + axes_a = self.axes_tuple + axes_b = second.axes_tuple + + kinds_a = dict() + kinds_b = dict() + + dimensions_pass = True + for axis_a, axis_b in zip(axes_a, axes_b): + kinds_a[axis_a.kind] = axis_a.size + kinds_b[axis_b.kind] = axis_b.size + if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list: + dimensions_pass = False + + if kinds_a.keys() != kinds_b.keys(): + return NeuralTypeComparisonResult.INCOMPATIBLE + for kind, size in kinds_a.items(): + if size != kinds_b[kind]: + return NeuralTypeComparisonResult.DIM_INCOMPATIBLE + + element_comparison_result = self.elements_type.compare(second.elements_type) + if dimensions_pass: + return element_comparison_result + elif element_comparison_result == NeuralTypeComparisonResult.SAME: + return NeuralTypeComparisonResult.TRANSPOSE_SAME + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + + def __check_sanity(self, axes): + # check that list come before any tensor dimension + are_strings = True + for axis in axes: + if not isinstance(axis, str): + are_strings = False + if isinstance(axis, str) and not are_strings: + raise ValueError("Either use full class names or all strings") + if are_strings: + return + checks_passed = True + saw_tensor_dim = False + for axis in axes: + if not axis.is_list: + saw_tensor_dim = True + else: # current axis is a list + if saw_tensor_dim: # which is preceded by tensor dim + checks_passed = False + if not checks_passed: + raise ValueError( + "You have list dimension after Tensor dimension. All list dimensions must preceed Tensor dimensions" + ) + + +class NmTensor(NeuralType): + """Class representing data which flows between NeuralModules' ports. + It also has a type of NeuralType represented by inheriting from NeuralType + object.""" + + def __init__(self, producer, producer_args, name, ntype=None): + """NmTensor constructor. + + Args: + producer (NeuralModule): object which produced this + producer_args (dict): a dictionary of port_name->NmTensor value + of arguments which were sent to producer to create this + """ + super(NmTensor, self).__init__(elements_type=ntype.elemts_type, axes=ntype.axes, optional=ntype.optional) + self._producer = producer + self._producer_args = producer_args + self._name = name + self._uuid = str(uuid.uuid4()) + + @property + def producer(self): + """ + Returns: + NeuralModule object which produced this NmTensor. + """ + return self._producer + + @property + def producer_args(self): + """ + Returns: + a dictionary of port_name->NmTensor value + of arguments which were sent to producer to create this object + """ + return self._producer_args + + @property + def name(self): + """ + Returns: + A NmTensor's name which should be equal to + the NeuralModule's output port's name which created it + """ + return self._name + + @property + def unique_name(self): + """Unique NMTensor name. + It is composed of non-unique name (self.name) and uuid of NeuralModule + which created this tensor. + + Returns: + str: unique name + """ + if self._producer is None: + raise ValueError("This NmTensor does not have a unique name") + return f"{self._name}~~~{self.producer}~~~{self._uuid}" + + +class NeuralTypeError(Exception): + """Base class for neural type related exceptions.""" + + pass + + +class NeuralPortNameMismatchError(NeuralTypeError): + """Exception raised when neural module is called with incorrect port + names.""" + + def __init__(self, message): + self.message = message + + +class NeuralPortNmTensorMismatchError(NeuralTypeError): + """Exception raised when a port is fed with a NmTensor of incompatible + type.""" + + def __init__(self, message): + self.message = message + + +class CanNotInferResultNeuralType(NeuralTypeError): + """Exception raised when NeuralType of output can not be inferred.""" + + def __init__(self, message): + self.message = message diff --git a/nemo/core/neural_types.py b/nemo/core/old_neural_types.py similarity index 100% rename from nemo/core/neural_types.py rename to nemo/core/old_neural_types.py From 21e7f319c69c82975be6c13102eaf2a2ad60d6e0 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 28 Jan 2020 17:01:23 -0800 Subject: [PATCH 02/70] fixing some unittests Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 2 +- nemo/backends/pytorch/common/rnn.py | 2 +- nemo/backends/pytorch/common/search.py | 2 +- nemo/core/__init__.py | 20 +++++- nemo/core/callbacks.py | 18 ++++- nemo/core/neural_factory.py | 18 ++++- nemo/core/neural_modules.py | 18 ++++- nemo/core/neural_types/__init__.py | 4 +- nemo/core/neural_types/elements.py | 53 ++++++++++++++- nemo/core/neural_types/neural_type.py | 3 +- tests/core/__init__.py | 0 tests/core/test_neural_types.py | 92 ++++++++++++++++++++++++++ 12 files changed, 219 insertions(+), 13 deletions(-) create mode 100644 tests/core/__init__.py create mode 100644 tests/core/test_neural_types.py diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index 295c09ba1ce4..90a20a633c81 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -2,7 +2,7 @@ from torch import nn from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, RegressionTag, TimeTag +from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, RegressionTag, TimeTag __all__ = ['SequenceLoss', 'CrossEntropyLoss', 'MSELoss'] diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index c7f6fc66f5bc..c171ad7e00fd 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -8,7 +8,7 @@ from nemo.backends.pytorch.common.parts import Attention from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag from nemo.utils.misc import pad_to diff --git a/nemo/backends/pytorch/common/search.py b/nemo/backends/pytorch/common/search.py index 812c22ce2cfd..7b449acdd0d3 100644 --- a/nemo/backends/pytorch/common/search.py +++ b/nemo/backends/pytorch/common/search.py @@ -3,7 +3,7 @@ import torch from nemo.backends.pytorch.nm import NonTrainableNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag INF = float('inf') BIG_NUM = 1e4 diff --git a/nemo/core/__init__.py b/nemo/core/__init__.py index 7b13691e476a..06a0050f1b7e 100644 --- a/nemo/core/__init__.py +++ b/nemo/core/__init__.py @@ -1,5 +1,21 @@ -# Copyright (c) 2019 NVIDIA Corporation +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .callbacks import * from .neural_factory import * from .neural_modules import * -from .neural_types import * +from .old_neural_types import * diff --git a/nemo/core/callbacks.py b/nemo/core/callbacks.py index 4f6c94ba01dc..1ebf3675e270 100644 --- a/nemo/core/callbacks.py +++ b/nemo/core/callbacks.py @@ -1,4 +1,20 @@ -# Copyright (c) 2019 NVIDIA Corporation +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import glob import os import sys diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 086af2a04fbf..9f61c086b58e 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -1,4 +1,20 @@ -# Copyright (c) 2019 NVIDIA Corporation +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + __all__ = [ 'Backend', 'ModelMode', diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 663bb3da3184..373839ee93b2 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -1,4 +1,20 @@ -# Copyright (c) 2019 NVIDIA Corporation +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """This file contains NeuralModule and NmTensor classes.""" __all__ = ['WeightShareTransform', 'NeuralModule'] diff --git a/nemo/core/neural_types/__init__.py b/nemo/core/neural_types/__init__.py index 92c9b37c32b6..124adc132c72 100644 --- a/nemo/core/neural_types/__init__.py +++ b/nemo/core/neural_types/__init__.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .elements import * from .axes import * from .comparison import * -from .neural_type import * \ No newline at end of file +from .elements import * +from .neural_type import * diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 0b3626556b96..b806280677f5 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -15,10 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['ElementType', 'VoidType'] +__all__ = [ + 'ElementType', + 'VoidType', + 'ChannelType', + 'AcousticEncodedRepresentation', + 'AudioSignal', + 'SpectrogramType', + 'MelSpectrogramType', + 'MFCCSpectrogramType', +] import abc from abc import ABC, abstractmethod -from typing import Tuple, Optional, Dict +from typing import Dict, Optional, Tuple + from .comparison import NeuralTypeComparisonResult @@ -81,3 +91,42 @@ def __str__(self): def compare(cls, second: abc.ABCMeta) -> NeuralTypeComparisonResult: return NeuralTypeComparisonResult.SAME + + +# TODO: Consider moving these files elsewhere +class ChannelType(ElementType): + def __str__(self): + return "convolutional channel value" + + +class AcousticEncodedRepresentation(ChannelType): + def __str__(self): + return "encoded representation returned by the acoustic encoder model" + + +class AudioSignal(ElementType): + def __str__(self): + return "encoded representation returned by the acoustic encoder model" + + def __init__(self, freq=16000): + self._params = {} + self._params['freq'] = freq + + @property + def type_parameters(self): + return self._params + + +class SpectrogramType(ChannelType): + def __str__(self): + return "generic spectorgram type" + + +class MelSpectrogramType(SpectrogramType): + def __str__(self): + return "mel spectorgram type" + + +class MFCCSpectrogramType(SpectrogramType): + def __str__(self): + return "mfcc spectorgram type" diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index cbb216ef80ef..346668e7d303 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -26,8 +26,9 @@ ] import uuid from typing import Tuple + +from .axes import AxisKind, AxisType from .comparison import NeuralTypeComparisonResult -from .axes import AxisType, AxisKind from .elements import * diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py new file mode 100644 index 000000000000..bffdf705bd56 --- /dev/null +++ b/tests/core/test_neural_types.py @@ -0,0 +1,92 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2019 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from nemo.core.neural_types import ( + AcousticEncodedRepresentation, + AudioSignal, + AxisKind, + AxisType, + ChannelType, + MelSpectrogramType, + MFCCSpectrogramType, + NeuralType, + NeuralTypeComparisonResult, + SpectrogramType, +) +from tests.common_setup import NeMoUnitTest + + +class NeuralTypeSystemTests(NeMoUnitTest): + def test_short_vs_long_version(self): + long_version = NeuralType( + elements_type=AcousticEncodedRepresentation(), + axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)), + ) + short_version = NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')) + self.assertEqual(long_version.compare(short_version), NeuralTypeComparisonResult.SAME) + self.assertEqual(short_version.compare(long_version), NeuralTypeComparisonResult.SAME) + + def test_parameterized_type_audio_sampling_frequency(self): + audio16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) + audio8K = NeuralType(AudioSignal(8000), axes=('B', 'T')) + another16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) + + self.assertEqual(audio8K.compare(audio16K), NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS) + self.assertEqual(audio16K.compare(audio8K), NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS) + self.assertEqual(another16K.compare(audio16K), NeuralTypeComparisonResult.SAME) + self.assertEqual(audio16K.compare(another16K), NeuralTypeComparisonResult.SAME) + + def test_transpose_same(self): + audio16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) + audio16K_t = NeuralType(AudioSignal(16000), axes=('T', 'B')) + self.assertEqual(audio16K.compare(audio16K_t), NeuralTypeComparisonResult.TRANSPOSE_SAME) + + def test_inheritance_spec_augment_example(self): + input = NeuralType(SpectrogramType(), ('B', 'D', 'T')) + out1 = NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) + out2 = NeuralType(MFCCSpectrogramType(), ('B', 'D', 'T')) + self.assertEqual(out1.compare(out2), NeuralTypeComparisonResult.INCOMPATIBLE) + self.assertEqual(out2.compare(out1), NeuralTypeComparisonResult.INCOMPATIBLE) + self.assertEqual(input.compare(out1), NeuralTypeComparisonResult.GREATER) + self.assertEqual(input.compare(out2), NeuralTypeComparisonResult.GREATER) + self.assertEqual(out1.compare(input), NeuralTypeComparisonResult.LESS) + self.assertEqual(out2.compare(input), NeuralTypeComparisonResult.LESS) + + def test_list_of_lists(self): + T1 = NeuralType( + elements_type=ChannelType(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + ) + T2 = NeuralType( + elements_type=ChannelType(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + ) + # TODO: should this be incompatible instead??? + self.assertEqual(T1.compare(T2), NeuralTypeComparisonResult.TRANSPOSE_SAME) From 881d4bb6e89d6c919753eaf884c585644793c30c Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Thu, 30 Jan 2020 14:19:38 -0800 Subject: [PATCH 03/70] fixing some files Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 33 +- nemo/backends/pytorch/common/rnn.py | 2 +- nemo/backends/pytorch/common/search.py | 2 +- nemo/core/__init__.py | 2 +- nemo/core/neural_types/elements.py | 24 ++ nemo/core/neural_types/neural_type.py | 33 +- nemo/core/old_neural_types.py | 410 ------------------------- tests/core/test_neural_types.py | 14 +- 8 files changed, 71 insertions(+), 449 deletions(-) delete mode 100644 nemo/core/old_neural_types.py diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index 90a20a633c81..1c2d4b2b1524 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -2,7 +2,7 @@ from torch import nn from nemo.backends.pytorch.nm import LossNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, RegressionTag, TimeTag +from nemo.core.neural_types import NeuralType, LogitsType, LabelsType, LossType, RegressionValuesType __all__ = ['SequenceLoss', 'CrossEntropyLoss', 'MSELoss'] @@ -34,23 +34,10 @@ class SequenceLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - log_probs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - targets: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - """ return { - 'log_probs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), - 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + 'log_probs': NeuralType(axes=('B', 'T', 'D')), + 'targets': NeuralType(axes=('B', 'T')) } @property @@ -61,7 +48,7 @@ def output_ports(self): NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(elements_type=LossType)} def __init__( self, @@ -139,8 +126,8 @@ def input_ports(self): """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "labels": NeuralType({0: AxisType(BatchTag),}), + "logits": NeuralType(elements_type=LogitsType, axes=('B', 'D')), + "labels": NeuralType(elements_type=LabelsType, axes=tuple('B')) } @property @@ -150,7 +137,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(elements_type=LossType)} def __init__(self, weight=None, **kwargs): LossNM.__init__(self, **kwargs) @@ -175,8 +162,8 @@ def input_ports(self): 0: AxisType(RegressionTag) """ return { - "preds": NeuralType({0: AxisType(RegressionTag)}), - "labels": NeuralType({0: AxisType(RegressionTag)}), + "preds": NeuralType(RegressionValuesType, tuple('B')), + "labels": NeuralType(LabelsType, tuple('B')), } @property @@ -186,7 +173,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(elements_type=LossType)} def __init__(self, **kwargs): LossNM.__init__(self, **kwargs) diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index c171ad7e00fd..247e8043879d 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -8,7 +8,7 @@ from nemo.backends.pytorch.common.parts import Attention from nemo.backends.pytorch.nm import TrainableNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import AxisType from nemo.utils.misc import pad_to diff --git a/nemo/backends/pytorch/common/search.py b/nemo/backends/pytorch/common/search.py index 7b449acdd0d3..7ddc8d553dd1 100644 --- a/nemo/backends/pytorch/common/search.py +++ b/nemo/backends/pytorch/common/search.py @@ -3,7 +3,7 @@ import torch from nemo.backends.pytorch.nm import NonTrainableNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import AxisType INF = float('inf') BIG_NUM = 1e4 diff --git a/nemo/core/__init__.py b/nemo/core/__init__.py index 06a0050f1b7e..e48567b139a6 100644 --- a/nemo/core/__init__.py +++ b/nemo/core/__init__.py @@ -18,4 +18,4 @@ from .callbacks import * from .neural_factory import * from .neural_modules import * -from .old_neural_types import * +from .neural_types import * diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index b806280677f5..b1a171e9507f 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -24,6 +24,10 @@ 'SpectrogramType', 'MelSpectrogramType', 'MFCCSpectrogramType', + 'LogitsType', + 'LabelsType', + 'LossType', + 'RegressionValuesType' ] import abc from abc import ABC, abstractmethod @@ -99,6 +103,21 @@ def __str__(self): return "convolutional channel value" +class LogitsType(ElementType): + def __str__(self): + return "neural type representing logits" + + +class LabelsType(ElementType): + def __str__(self): + return "neural type representing labels" + + +class LossType(ElementType): + def __str__(self): + return "neural type representing loss value" + + class AcousticEncodedRepresentation(ChannelType): def __str__(self): return "encoded representation returned by the acoustic encoder model" @@ -130,3 +149,8 @@ def __str__(self): class MFCCSpectrogramType(SpectrogramType): def __str__(self): return "mfcc spectorgram type" + + +class RegressionValuesType(ElementType): + def __str__(self): + return "regression values type" diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 346668e7d303..fede950785b7 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -25,7 +25,7 @@ 'CanNotInferResultNeuralType', ] import uuid -from typing import Tuple +from typing import Tuple, Optional from .axes import AxisKind, AxisType from .comparison import NeuralTypeComparisonResult @@ -36,18 +36,21 @@ class NeuralType(object): """This is the main class which would represent neural type concept. nmTensors derives from this. It is used to represent *the types* of inputs and outputs.""" - def __init__(self, elements_type: ElementType, axes: Tuple, optional=False): - self.__check_sanity(axes) + def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple] = None, optional=False): self.elements_type = elements_type - axes_list = [] - for axis in axes: - if isinstance(axis, str): - axes_list.append(AxisType(AxisKind.from_str(axis), None)) - elif isinstance(axis, AxisType): - axes_list.append(axis) - else: - raise ValueError(f"axis type must be either str or AxisType instance") - self.axes_tuple = tuple(axes_list) + if axes is not None: + self.__check_sanity(axes) + axes_list = [] + for axis in axes: + if isinstance(axis, str): + axes_list.append(AxisType(AxisKind.from_str(axis), None)) + elif isinstance(axis, AxisType): + axes_list.append(axis) + else: + raise ValueError(f"axis type must be either str or AxisType instance") + self.axes_tuple = tuple(axes_list) + else: + self.axes_tuple = None self.optional = optional def compare(self, second) -> NeuralTypeComparisonResult: @@ -58,6 +61,12 @@ def compare(self, second) -> NeuralTypeComparisonResult: kinds_a = dict() kinds_b = dict() + if self.axes_tuple is None: + if second.axes_tuple is None: + return self.elements_type.compare(second.elements_type) + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + dimensions_pass = True for axis_a, axis_b in zip(axes_a, axes_b): kinds_a[axis_a.kind] = axis_a.size diff --git a/nemo/core/old_neural_types.py b/nemo/core/old_neural_types.py deleted file mode 100644 index 38b606fc5b9e..000000000000 --- a/nemo/core/old_neural_types.py +++ /dev/null @@ -1,410 +0,0 @@ -# Copyright (c) 2019 NVIDIA Corporation -"""This module contains Tags, AxisTypes, NeuralTypes and NmTensors. -Every NmTensor is of a particular Neural Type. -Neural Modules' input and output ports are also of Neural Type. - -An exception will be raised when a NmTensor and input port where it goes are -of incompatible types. -""" -__all__ = [ - 'BaseTag', - 'BatchTag', - 'TimeTag', - 'ProcessedTimeTag', - 'ChannelTag', - 'EmbeddedTextTag', - 'SpectrogramSignalTag', - 'MelSpectrogramSignalTag', - 'MFCCSignalTag', - 'EncodedRepresentationTag', - 'ClassTag', - 'WidthTag', - 'HeightTag', - 'CategoricalTag', - 'RegressionTag', - 'NeuralTypeComparisonResult', - 'AxisType', - 'NeuralType', - 'NmTensor', - 'NeuralTypeError', - 'NeuralPortNameMismatchError', - 'NeuralPortNmTensorMismatchError', - 'CanNotInferResultNeuralType', -] - -import uuid -from enum import Enum - - -class BaseTag(object): - """Base Neural Tag. All Tags should inherit from this.""" - - def __str__(self): - return "base" - - -class BatchTag(BaseTag): - """Tag for batch dimension.""" - - def __str__(self): - return "batch" - - -class TimeTag(BaseTag): - """Tag for time dimension.""" - - def __str__(self): - return "time" - - -class ProcessedTimeTag(TimeTag): - """Tag for processed time dimension. - For example: after pre-processing, or augmentation.""" - - def __str__(self): - return "processed_time" - - -class ChannelTag(BaseTag): - """Tag for channel dimension.""" - - def __str__(self): - return "channel" - - -class EmbeddedTextTag(ChannelTag): - """Tag for any dimensions that contains text that goes through an - enbedding layer.""" - - def __str__(self): - return "embedded_text" - - -class SpectrogramSignalTag(ChannelTag): - """Tag for spectrogram signal dimension.""" - - def __str__(self): - return "spectrogram_signal" - - -class MelSpectrogramSignalTag(SpectrogramSignalTag): - """Tag for mel spectrogram signal dimension.""" - - def __str__(self): - return "mel_spectrogram_signal" - - -class MFCCSignalTag(SpectrogramSignalTag): - """Tag for MFCC signal dimension.""" - - def __str__(self): - return "mfcc_signal" - - -class EncodedRepresentationTag(ChannelTag): - """Tag for encoded representation. This should be used to - denote encoders' outputs.""" - - def __str__(self): - return "encoded_representation" - - -class ClassTag(BaseTag): - """Tag for class dimension. - For example, number of classes in classification problem, - vocabuary size or num of characters for ASR.""" - - def __str__(self): - return "channel" - - -class WidthTag(BaseTag): - """Tag for width dimension.""" - - def __str__(self): - return "width" - - -class HeightTag(BaseTag): - """Tag for width dimension.""" - - def __str__(self): - return "height" - - -class CategoricalTag(BatchTag): - """Tag for labels for classification tasks.""" - - def __str__(self): - return "category" - - -class RegressionTag(BatchTag): - """Tag for labels for regression tasks. - For example, this should be used in STS-B task, where labels - represent semantic semilarity score (float)""" - - def __str__(self): - return "regression" - - -class NeuralTypeComparisonResult(Enum): - """The result of comparing two neural type objects for compatibility. - When comparing A.compare_to(B):""" - - SAME = 0 - LESS = 1 # A is B - GREATER = 2 # B is A - DIM_INCOMPATIBLE = 3 # Resize connector might fix incompatibility - TRANSPOSE_SAME = 4 # A transpose will make them same - INCOMPATIBLE = 5 # A and B are incompatible. Can't fix incompatibility automatically - - -class AxisType(object): - """Every tensor's axis has semantics, dimension and descriptor. - It's semantics is a Neural Tag (inherited from BaseTag) - dimension (dim) is (optional) int and descriptor is (optional) string""" - - def __init__(self, semantics, dim: int = None, descriptor: str = None): - self._semantics = semantics - self._dim = dim - self._descriptor = descriptor - - def __eq__(self, other): - return self.semantics == other.semantics and self.dim == other.dim and self.descriptor == other.descriptor - - def __str__(self): - return "{0}:{1}:{2}".format(self.semantics, self.dim, self.descriptor) - - def __hash__(self): - return hash(self.__str__()) - - def compare_to(self, other): - """ - Compares current AxisType object to other - - Args: - other (AxisType): other AxisType object to compare with - - Returns: - Results of a comparison (NeuralTypeComparisonResult) - """ - if (self.dim is None or self.dim == other.dim) and self.descriptor == other.descriptor: - if self.semantics == other.semantics: - return NeuralTypeComparisonResult.SAME - elif issubclass(self.semantics, other.semantics): - return NeuralTypeComparisonResult.LESS - elif issubclass(other.semantics, self.semantics): - return NeuralTypeComparisonResult.GREATER - else: - return NeuralTypeComparisonResult.INCOMPATIBLE - elif self.descriptor == other.descriptor and self.semantics == other.semantics: - return NeuralTypeComparisonResult.DIM_INCOMPATIBLE - else: - return NeuralTypeComparisonResult.INCOMPATIBLE - - @property - def semantics(self): - return self._semantics - - @property - def dim(self): - return self._dim - - @property - def descriptor(self): - return self._descriptor - - -class NeuralType(object): - """Neural Type: a type for NmTensor. - - Note: This type mechanism is represented by Python inheritance. That is, - NmTensor - class inherits from NeuralType class. - - A Neural Type is a mapping from Tensor's axis number to it's type ( - AxisType). - - To instantiate a NeuralType you should pass it a dictionary (axis2type) - which - will map axis to it's AxisType. You can also pass optional argument when - describing input ports. - - For example, a ResNet18 input can be described as: - - .. code-block:: python - - NeuralType({0: AxisType(BatchTag, None, None), - 1: AxisType(ChannelTag, None, None), - 2: AxisType(HeightTag, 224, None), - 3: AxisType(WidthTag, 224, None)}) - - Special cases: - - non-tensor objects should be denoted as NeuralType(None) - - root type is denoted by NeuralType({}). A port of NeuralType({}) must - - accept NmTensors of any NeuralType. More specifically: - root_type = NeuralType({}) - root_type.compare(any_other_neural_type) == - NeuralTypeComparisonResult.SAME - - - See "nemo/tests/test_neural_types.py" for more examples. - - """ - - # def __init__(self, axis2type=None): - def __init__(self, axis2type={}, optional=False): - """ - Constructor - Args: - axis2type: mapping axises to it's AxisType - optional: (default: False). If this port is optional - """ - self._axis2type = axis2type - self._optional = optional - - def __str__(self): - if self._axis2type is None: - return "(Optional) " if self._optional else "" + "non-tensor " "object" - elif len(self._axis2type) == 0: - return "(Optional) " if self._optional else "" + "Root NeuralType" - return ( - "(Optional)" - if self._optional - else "" + "\n".join(["{0}->{1}".format(axis, tag) for axis, tag in self._axis2type.items()]) - ) - - def compare(self, n_type2) -> NeuralTypeComparisonResult: - """Compares if current object's NeuralType semantics is compatible - with n_type2 - - Args: - n_type2 (NeuralType): a type to compare with - - Returns: - Results of a comparison (NeuralTypeComparisonResult) - """ - # self is a root type - if self.axis2type is not None and len(self.axis2type) == 0: - return NeuralTypeComparisonResult.SAME - # n_type2 is root type but self is not - elif n_type2.axis2type is not None and len(n_type2.axis2type) == 0: - return NeuralTypeComparisonResult.INCOMPATIBLE - # one is None while other is not - elif self._axis2type is None and n_type2._axis2type is not None: - return NeuralTypeComparisonResult.INCOMPATIBLE - elif self._axis2type is not None and n_type2._axis2type is None: - return NeuralTypeComparisonResult.INCOMPATIBLE - # same neural type - elif self._axis2type == n_type2._axis2type: - return NeuralTypeComparisonResult.SAME - # same set of keys and set of values => TRANSPOSE_SAME - elif set(self._axis2type.keys()) == set(n_type2._axis2type.keys()) and set(self._axis2type.values()) == set( - n_type2._axis2type.values() - ): - return NeuralTypeComparisonResult.TRANSPOSE_SAME - - elif set(self._axis2type.keys()) == set(n_type2._axis2type.keys()): - # comparison_result = 1 - comparison_result = 0 - for key in self._axis2type.keys(): - comparison_result = max( - self._axis2type[key].compare_to(n_type2._axis2type[key]).value, comparison_result, - ) - return NeuralTypeComparisonResult(comparison_result) - else: - return NeuralTypeComparisonResult.INCOMPATIBLE - - @property - def axis2type(self): - return self._axis2type - - -class NmTensor(NeuralType): - """Class representing data which flows between NeuralModules' ports. - It also has a type of NeuralType represented by inheriting from NeuralType - object.""" - - def __init__(self, producer, producer_args, name, ntype=None): - """NmTensor constructor. - - Args: - producer (NeuralModule): object which produced this - producer_args (dict): a dictionary of port_name->NmTensor value - of arguments which were sent to producer to create this - """ - super(NmTensor, self).__init__(axis2type=ntype._axis2type) - self._producer = producer - self._producer_args = producer_args - self._name = name - self._uuid = str(uuid.uuid4()) - - @property - def producer(self): - """ - Returns: - NeuralModule object which produced this NmTensor. - """ - return self._producer - - @property - def producer_args(self): - """ - Returns: - a dictionary of port_name->NmTensor value - of arguments which were sent to producer to create this object - """ - return self._producer_args - - @property - def name(self): - """ - Returns: - A NmTensor's name which should be equal to - the NeuralModule's output port's name which created it - """ - return self._name - - @property - def unique_name(self): - """Unique NMTensor name. - It is composed of non-unique name (self.name) and uuid of NeuralModule - which created this tensor. - - Returns: - str: unique name - """ - if self._producer is None: - raise ValueError("This NmTensor does not have a unique name") - return f"{self._name}~~~{self.producer}~~~{self._uuid}" - - -class NeuralTypeError(Exception): - """Base class for neural type related exceptions.""" - - pass - - -class NeuralPortNameMismatchError(NeuralTypeError): - """Exception raised when neural module is called with incorrect port - names.""" - - def __init__(self, message): - self.message = message - - -class NeuralPortNmTensorMismatchError(NeuralTypeError): - """Exception raised when a port is fed with a NmTensor of incompatible - type.""" - - def __init__(self, message): - self.message = message - - -class CanNotInferResultNeuralType(NeuralTypeError): - """Exception raised when NeuralType of output can not be inferred.""" - - def __init__(self, message): - self.message = message diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index bffdf705bd56..51a1c9b18044 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -51,7 +51,13 @@ def test_parameterized_type_audio_sampling_frequency(self): self.assertEqual(another16K.compare(audio16K), NeuralTypeComparisonResult.SAME) self.assertEqual(audio16K.compare(another16K), NeuralTypeComparisonResult.SAME) - def test_transpose_same(self): + def test_transpose_same_1(self): + type1 = NeuralType(axes=('B', 'T', 'C')) + type2 = NeuralType(axes=('T', 'B', 'C')) + self.assertEqual(type1.compare(type2), NeuralTypeComparisonResult.TRANSPOSE_SAME) + self.assertEqual(type2.compare(type1), NeuralTypeComparisonResult.TRANSPOSE_SAME) + + def test_transpose_same_2(self): audio16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) audio16K_t = NeuralType(AudioSignal(16000), axes=('T', 'B')) self.assertEqual(audio16K.compare(audio16K_t), NeuralTypeComparisonResult.TRANSPOSE_SAME) @@ -67,6 +73,12 @@ def test_inheritance_spec_augment_example(self): self.assertEqual(out1.compare(input), NeuralTypeComparisonResult.LESS) self.assertEqual(out2.compare(input), NeuralTypeComparisonResult.LESS) + def test_singletone(self): + loss_output1 = NeuralType(axes=None) + loss_output2 = NeuralType(axes=None) + self.assertEqual(loss_output1.compare(loss_output2), NeuralTypeComparisonResult.SAME) + self.assertEqual(loss_output2.compare(loss_output1), NeuralTypeComparisonResult.SAME) + def test_list_of_lists(self): T1 = NeuralType( elements_type=ChannelType(), From b8f633fba19b9afdb13e94e84a800ae0e9f27dd7 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 31 Jan 2020 13:05:03 -0800 Subject: [PATCH 04/70] simplest examples working Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/other.py | 10 +- .../pytorch/tutorials/chatbot/data.py | 2 +- .../pytorch/tutorials/chatbot/modules.py | 138 +++-------------- nemo/backends/pytorch/tutorials/toys.py | 140 +++--------------- nemo/core/neural_modules.py | 54 ++++--- nemo/core/neural_types/neural_type.py | 20 ++- tests/core/test_neural_modules.py | 57 +++++++ tests/{ => core}/test_pytorch_trainers.py | 0 8 files changed, 142 insertions(+), 279 deletions(-) create mode 100644 tests/core/test_neural_modules.py rename tests/{ => core}/test_pytorch_trainers.py (100%) diff --git a/nemo/backends/pytorch/common/other.py b/nemo/backends/pytorch/common/other.py index 982abd100446..408dc40613dc 100644 --- a/nemo/backends/pytorch/common/other.py +++ b/nemo/backends/pytorch/common/other.py @@ -33,14 +33,8 @@ class SimpleCombiner(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - x1: - Empty?!? - - x2: - Empty?!? """ - return {"x1": NeuralType({}), "x2": NeuralType({})} + return {"x1": NeuralType(VoidType()), "x2": NeuralType(VoidType())} @property def output_ports(self): @@ -49,7 +43,7 @@ def output_ports(self): combined: None """ - return {"combined": None} + return {"combined": NeuralType(VoidType())} def __init__(self, mode="add", **kwargs): TrainableNM.__init__(self, **kwargs) diff --git a/nemo/backends/pytorch/tutorials/chatbot/data.py b/nemo/backends/pytorch/tutorials/chatbot/data.py index a4ea9124e4cb..6f53877c7754 100644 --- a/nemo/backends/pytorch/tutorials/chatbot/data.py +++ b/nemo/backends/pytorch/tutorials/chatbot/data.py @@ -213,7 +213,7 @@ def outputVar(l, voc): max_target_len = max([len(indexes) for indexes in indexes_batch]) padList = zeroPadding(indexes_batch) mask = binaryMatrix(padList) - mask = t.ByteTensor(mask) + mask = t.ByteTensor(mask).to(t.bool) padVar = t.LongTensor(padList) return padVar, mask, max_target_len diff --git a/nemo/backends/pytorch/tutorials/chatbot/modules.py b/nemo/backends/pytorch/tutorials/chatbot/modules.py index de98c5799edb..0386a2560323 100644 --- a/nemo/backends/pytorch/tutorials/chatbot/modules.py +++ b/nemo/backends/pytorch/tutorials/chatbot/modules.py @@ -20,34 +20,13 @@ class DialogDataLayer(DataLayerNM): @property def output_ports(self): """Returns definitions of module output ports. - - src: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - src_lengths: - 0: AxisType(BatchTag) - - tgt: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - mask: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - max_tgt_lengths: - None """ return { - "src": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "src_lengths": NeuralType({0: AxisType(BatchTag)}), - "tgt": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "mask": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "max_tgt_lengths": NeuralType(None), + "src": NeuralType(ChannelType(), ('T', 'B')), + "src_lengths": NeuralType(ChannelType(), tuple('B')), + "tgt": NeuralType(LabelsType(), ('T', 'B')), + "mask": NeuralType(ChannelType(), ('T', 'B')), + "max_tgt_lengths": NeuralType(axes=None), } def __init__(self, *, batch_size, corpus_name, datafile, min_count=3, **kwargs): @@ -94,39 +73,19 @@ class EncoderRNN(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_seq: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - input_lengths: - 0: AxisType(BatchTag) """ return { - "input_seq": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "input_lengths": NeuralType({0: AxisType(BatchTag)}), + "input_seq": NeuralType(ChannelType(), ('T', 'B')), + "input_lengths": NeuralType(ChannelType(), tuple('B')), } @property def output_ports(self): """Returns definitions of module output ports. - - outputs: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - 2: AxisType(ChannelTag) - - hidden: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),}), - "hidden": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), + "hidden": NeuralType(ChannelType(), ('B', 'D')), } def __init__(self, *, voc_size, encoder_n_layers, hidden_size, dropout, bidirectional=True, **kwargs): @@ -174,26 +133,11 @@ class LuongAttnDecoderRNN(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - targets: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - encoder_outputs: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - 2: AxisType(ChannelTag) - - max_target_len: - None """ return { - "targets": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "encoder_outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),}), - "max_target_len": NeuralType(None), + "targets": NeuralType(LabelsType(), ('T', 'B')), + "encoder_outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), + "max_target_len": NeuralType(axes=None), } @property @@ -213,8 +157,8 @@ def output_ports(self): 1: AxisType(ChannelTag) """ return { - "outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),}), - "hidden": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), + "hidden": NeuralType(ChannelType(), ('B', 'D')), } def __init__(self, *, attn_model, hidden_size, voc_size, decoder_n_layers, dropout, **kwargs): @@ -327,28 +271,11 @@ class MaskedXEntropyLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - predictions - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - 2: AxisType(ChannelTag)} - - target: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - mask: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) """ return { - "predictions": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),}), - "target": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), - "mask": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)}), + "predictions": NeuralType(ChannelType(), ('T', 'B', 'D')), + "target": NeuralType(LabelsType(), ('T', 'B')), + "mask": NeuralType(ChannelType(), ('T', 'B')), } @property @@ -358,7 +285,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType(), axes=None)} def __init__(self, **kwargs): LossNM.__init__(self, **kwargs) @@ -381,39 +308,16 @@ class GreedyLuongAttnDecoderRNN(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - encoder_outputs: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - 2: AxisType(ChannelTag) """ - return {"encoder_outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),})} + return {"encoder_outputs": NeuralType(ChannelType(), ('T', 'B', 'D'))} @property def output_ports(self): """Returns definitions of module output ports. - - outputs: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - hidden: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "outputs": NeuralType( - { - 0: AxisType(TimeTag), - 1: AxisType(BatchTag), - # 2: AxisType(ChannelTag) - } - ), - "hidden": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "outputs": NeuralType(ChannelType(), ('T', 'B')), + "hidden": NeuralType(ChannelType(), ('B', 'D')), } def __init__(self, *, attn_model, hidden_size, voc_size, decoder_n_layers, dropout, max_dec_steps=10, **kwargs): diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index b2449c5ddfd5..55e83aaf986d 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -6,7 +6,7 @@ import torch.utils.data as t_utils from ....core import DeviceType, NeuralModule -from ....core.neural_types import * +from ....core.neural_types import NeuralType, ChannelType, LabelsType from ..nm import DataLayerNM, LossNM, TrainableNM @@ -20,7 +20,7 @@ def input_ports(self): Returns: A (dict) of module's input ports names to NeuralTypes mapping """ - return {"x": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} + return {"x": NeuralType(ChannelType(), ('B', 'D'))} @property def output_ports(self): @@ -29,7 +29,7 @@ def output_ports(self): Returns: A (dict) of module's output ports names to NeuralTypes mapping """ - return {"y_pred": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} + return {"y_pred": NeuralType(ChannelType(), ('B', 'D'))} def __init__(self, *, dim, **kwargs): # Part specific for Neural Modules API: @@ -60,31 +60,17 @@ class TaylorNetO(TrainableNM): # Note inheritance from TrainableNM def input_ports(self): """Returns definitions of module input ports. - x: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - o: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "x": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "o": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "x": NeuralType(ChannelType(), ('B', 'D')), + "o": NeuralType(ChannelType(), ('B', 'D')), } @property def output_ports(self): """Returns definitions of module output ports. - - y_pred: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ - return {"y_pred": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}, optional=True)} + return {"y_pred": NeuralType(ChannelType(), ('B', 'D'), optional=True)} def __init__(self, *, dim, **kwargs): # Part specific for Neural Modules API: @@ -133,20 +119,10 @@ def __len__(self): @property def output_ports(self): """Returns definitions of module output ports - - x: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - y: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "x": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "y": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "x": NeuralType(ChannelType(), ('B', 'D')), + "y": NeuralType(LabelsType(), ('B', 'D')), } def __init__(self, *, n, batch_size, f=t.sin, x_lo=-4, x_hi=4, **kwargs): @@ -188,8 +164,8 @@ def input_ports(self): 1: AxisType(ChannelTag) """ return { - "predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "target": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "predictions": NeuralType(ChannelType(), ('B', 'D')), + "target": NeuralType(LabelsType(), ('B', 'D')), } @property @@ -199,7 +175,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(axes=None)} def __init__(self, **kwargs): LossNM.__init__(self, **kwargs) @@ -213,20 +189,10 @@ class L1Loss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - predictions: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - target: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "target": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "predictions": NeuralType(ChannelType(), ('B', 'D')), + "target": NeuralType(LabelsType(), ('B', 'D')), } @property @@ -236,7 +202,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(axes=None)} def __init__(self, **kwargs): LossNM.__init__(self, **kwargs) @@ -250,18 +216,10 @@ class CrossEntropyLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - predictions: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - labels: - 0: AxisType(BatchTag) """ return { - "predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "labels": NeuralType({0: AxisType(BatchTag)}), + "predictions": NeuralType(ChannelType(), ('B', 'D')), + "labels": NeuralType(LabelsType(), tuple('B')), } @property @@ -271,7 +229,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(axes=None)} def __init__(self, **kwargs): # Neural Module API specific @@ -282,67 +240,3 @@ def __init__(self, **kwargs): # You need to implement this function def _loss_function(self, **kwargs): return self._criterion(*(kwargs.values())) - - -class DopeDualLoss(LossNM): - """ - The dual loss function that DOPE uses - """ - - @property - def input_ports(self): - """Returns definitions of module input ports. - - belief_predictions: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - belief_labels: - 0: AxisType(BatchTag) - - affinity_predictions: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - affinity_labels: - 0: AxisType(BatchTag) - """ - return { - "belief_predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "belief_labels": NeuralType({0: AxisType(BatchTag)}), - "affinity_predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "affinity_labels": NeuralType({0: AxisType(BatchTag)}), - } - - @property - def output_ports(self): - """Returns definitions of module output ports. - - loss: - NeuralType(None) - """ - return {"loss": NeuralType(None)} - - def __init__(self, **kwargs): - # Neural Module API specific - NeuralModule.__init__(self, **kwargs) - - # You need to implement this function - def _loss_function(self, **kwargs): - loss = 0.0 - - # Belief maps loss - # output, each belief map layers. - for l in kwargs["belief_predictions"]: - loss_tmp = ((l - kwargs["belief_labels"]) * (l - kwargs["belief_labels"])).mean() - loss += loss_tmp - - # Affinities loss - # output, each belief map layers. - for l in kwargs["affinity_predictions"]: - loss_tmp = ((l - kwargs["affinity_labels"]) * (l - kwargs["affinity_labels"])).mean() - loss += loss_tmp - - return loss diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 373839ee93b2..20faafbbaf19 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -22,6 +22,7 @@ import uuid from abc import ABC, abstractmethod from collections import namedtuple +from copy import deepcopy from enum import Enum from inspect import getargvalues, stack from typing import Dict, List, Optional, Set, Tuple @@ -142,10 +143,6 @@ def __call__(self, **kwargs): Returns: NmTensor object or tuple of NmTensor objects """ - # if self._assigned_top_order is not None: - # raise ValueError("We currently do not support calling same NM" - # "more than once") - # Get input and output ports definitions. input_port_defs = self.input_ports output_port_defs = self.output_ports @@ -153,34 +150,45 @@ def __call__(self, **kwargs): first_input_nmtensor_type = None input_nmtensors_are_of_same_type = True for port_name, tgv in kwargs.items(): + # make sure that passed arguments correspond to input port names if port_name not in input_port_defs.keys(): raise NeuralPortNameMismatchError("Wrong input port name: {0}".format(port_name)) - type_comatibility = input_port_defs[port_name].compare(tgv) - - if first_input_nmtensor_type is None: - first_input_nmtensor_type = NeuralType(tgv._axis2type) - else: - if first_input_nmtensor_type._axis2type is None: - input_nmtensors_are_of_same_type = True - else: - input_nmtensors_are_of_same_type = first_input_nmtensor_type.compare( - tgv - ) == NeuralTypeComparisonResult.SAME and len(first_input_nmtensor_type._axis2type) - if not ( - type_comatibility == NeuralTypeComparisonResult.SAME - or type_comatibility == NeuralTypeComparisonResult.GREATER - ): + input_port = input_port_defs[port_name] + type_comatibility = input_port.compare(tgv) + if type_comatibility != NeuralTypeComparisonResult.SAME and type_comatibility != \ + NeuralTypeComparisonResult.GREATER: raise NeuralPortNmTensorMismatchError( "\n\nIn {0}. \n" "Port: {1} and a NmTensor it was fed are \n" "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" "\n\nType comparison result: {4}".format( self.__class__.__name__, port_name, input_port_defs[port_name], tgv, type_comatibility, - ) - ) - if type_comatibility == NeuralTypeComparisonResult.LESS: - print('Types were raised') + )) + + # if first_input_nmtensor_type is None: + # first_input_nmtensor_type = NeuralType(tgv._axis2type) + # else: + # if first_input_nmtensor_type._axis2type is None: + # input_nmtensors_are_of_same_type = True + # else: + # input_nmtensors_are_of_same_type = first_input_nmtensor_type.compare( + # tgv + # ) == NeuralTypeComparisonResult.SAME and len(first_input_nmtensor_type._axis2type) + # if not ( + # type_comatibility == NeuralTypeComparisonResult.SAME + # or type_comatibility == NeuralTypeComparisonResult.GREATER + # ): + # raise NeuralPortNmTensorMismatchError( + # "\n\nIn {0}. \n" + # "Port: {1} and a NmTensor it was fed are \n" + # "of incompatible neural types:\n\n{2} \n\n and \n\n{3}" + # "\n\nType comparison result: {4}".format( + # self.__class__.__name__, port_name, input_port_defs[port_name], tgv, type_comatibility, + # ) + # ) + # if type_comatibility == NeuralTypeComparisonResult.LESS: + # print('Types were raised') if len(output_port_defs) == 1: out_name = list(output_port_defs)[0] diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index fede950785b7..83d12ba179a8 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -37,6 +37,9 @@ class NeuralType(object): nmTensors derives from this. It is used to represent *the types* of inputs and outputs.""" def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple] = None, optional=False): + if not isinstance(elements_type, ElementType): + raise ValueError(f"elements_type of NeuralType must be an instance of a class derived from ElementType." + f"Did you pass a class instead?") self.elements_type = elements_type if axes is not None: self.__check_sanity(axes) @@ -48,21 +51,24 @@ def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple axes_list.append(axis) else: raise ValueError(f"axis type must be either str or AxisType instance") - self.axes_tuple = tuple(axes_list) + self.axes = tuple(axes_list) else: - self.axes_tuple = None + self.axes = None self.optional = optional def compare(self, second) -> NeuralTypeComparisonResult: # First, handle dimensionality - axes_a = self.axes_tuple - axes_b = second.axes_tuple + axes_a = self.axes + axes_b = second.axes kinds_a = dict() kinds_b = dict() - if self.axes_tuple is None: - if second.axes_tuple is None: + if isinstance(self.elements_type, VoidType) and self.axes is None: + return NeuralTypeComparisonResult.SAME + + if self.axes is None: + if second.axes is None: return self.elements_type.compare(second.elements_type) else: return NeuralTypeComparisonResult.INCOMPATIBLE @@ -125,7 +131,7 @@ def __init__(self, producer, producer_args, name, ntype=None): producer_args (dict): a dictionary of port_name->NmTensor value of arguments which were sent to producer to create this """ - super(NmTensor, self).__init__(elements_type=ntype.elemts_type, axes=ntype.axes, optional=ntype.optional) + super(NmTensor, self).__init__(elements_type=ntype.elements_type, axes=ntype.axes, optional=ntype.optional) self._producer = producer self._producer_args = producer_args self._name = name diff --git a/tests/core/test_neural_modules.py b/tests/core/test_neural_modules.py new file mode 100644 index 000000000000..7b8a39bdf405 --- /dev/null +++ b/tests/core/test_neural_modules.py @@ -0,0 +1,57 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2019 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import nemo +from nemo.core.neural_types import NeuralType, ChannelType +from tests.common_setup import NeMoUnitTest + + +class NeuralModulesTests(NeMoUnitTest): + def test_call_TaylorNet(self): + x_tg = nemo.core.neural_modules.NmTensor( + producer=None, + producer_args=None, + name=None, + ntype=NeuralType(ChannelType(), ('B', 'D')) + ) + + tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) + # note that real port's name: x was used + y_pred = tn(x=x_tg) + self.assertEqual(y_pred.producer, tn) + self.assertEqual(y_pred.producer_args.get("x"), x_tg) + + def test_simplest_example_chain(self): + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=1) + trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) + loss = nemo.backends.pytorch.tutorials.MSELoss() + x, y = data_source() + y_pred = trainable_module(x=x) + loss_tensor = loss(predictions=y_pred, target=y) + + # check producers' bookkeeping + self.assertEqual(loss_tensor.producer, loss) + self.assertEqual(loss_tensor.producer_args, {"predictions": y_pred, "target": y}) + self.assertEqual(y_pred.producer, trainable_module) + self.assertEqual(y_pred.producer_args, {"x": x}) + self.assertEqual(y.producer, data_source) + self.assertEqual(y.producer_args, {}) + self.assertEqual(x.producer, data_source) + self.assertEqual(x.producer_args, {}) + + diff --git a/tests/test_pytorch_trainers.py b/tests/core/test_pytorch_trainers.py similarity index 100% rename from tests/test_pytorch_trainers.py rename to tests/core/test_pytorch_trainers.py From 0cec89514dd143e38e0106d1a71d6ba9aeebbf10 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 3 Feb 2020 16:52:47 -0800 Subject: [PATCH 05/70] fix codestyle Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 9 +++------ nemo/core/neural_types/elements.py | 2 +- nemo/core/neural_types/neural_type.py | 8 +++++--- tests/core/test_neural_modules.py | 11 +++-------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index dbed3ff5ee0c..633eee772b66 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -2,7 +2,7 @@ from torch import nn from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import NeuralType, LogitsType, LabelsType, LossType, RegressionValuesType +from nemo.core.neural_types import LabelsType, LogitsType, LossType, NeuralType, RegressionValuesType __all__ = ['SequenceLoss', 'CrossEntropyLoss', 'MSELoss'] @@ -35,10 +35,7 @@ class SequenceLoss(LossNM): def input_ports(self): """Returns definitions of module input ports. """ - return { - 'log_probs': NeuralType(axes=('B', 'T', 'D')), - 'targets': NeuralType(axes=('B', 'T')) - } + return {'log_probs': NeuralType(axes=('B', 'T', 'D')), 'targets': NeuralType(axes=('B', 'T'))} @property def output_ports(self): @@ -120,7 +117,7 @@ def input_ports(self): """ return { "logits": NeuralType(elements_type=LogitsType, axes=('B', 'D')), - "labels": NeuralType(elements_type=LabelsType, axes=tuple('B')) + "labels": NeuralType(elements_type=LabelsType, axes=tuple('B')), } @property diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index b1a171e9507f..f4c4d12445b4 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -27,7 +27,7 @@ 'LogitsType', 'LabelsType', 'LossType', - 'RegressionValuesType' + 'RegressionValuesType', ] import abc from abc import ABC, abstractmethod diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 83d12ba179a8..a2df777c9296 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -25,7 +25,7 @@ 'CanNotInferResultNeuralType', ] import uuid -from typing import Tuple, Optional +from typing import Optional, Tuple from .axes import AxisKind, AxisType from .comparison import NeuralTypeComparisonResult @@ -38,8 +38,10 @@ class NeuralType(object): def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple] = None, optional=False): if not isinstance(elements_type, ElementType): - raise ValueError(f"elements_type of NeuralType must be an instance of a class derived from ElementType." - f"Did you pass a class instead?") + raise ValueError( + f"elements_type of NeuralType must be an instance of a class derived from ElementType." + f"Did you pass a class instead?" + ) self.elements_type = elements_type if axes is not None: self.__check_sanity(axes) diff --git a/tests/core/test_neural_modules.py b/tests/core/test_neural_modules.py index 7b8a39bdf405..92dd80237d91 100644 --- a/tests/core/test_neural_modules.py +++ b/tests/core/test_neural_modules.py @@ -17,18 +17,15 @@ # ============================================================================= import nemo -from nemo.core.neural_types import NeuralType, ChannelType +from nemo.core.neural_types import ChannelType, NeuralType from tests.common_setup import NeMoUnitTest class NeuralModulesTests(NeMoUnitTest): def test_call_TaylorNet(self): x_tg = nemo.core.neural_modules.NmTensor( - producer=None, - producer_args=None, - name=None, - ntype=NeuralType(ChannelType(), ('B', 'D')) - ) + producer=None, producer_args=None, name=None, ntype=NeuralType(ChannelType(), ('B', 'D')) + ) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) # note that real port's name: x was used @@ -53,5 +50,3 @@ def test_simplest_example_chain(self): self.assertEqual(y.producer_args, {}) self.assertEqual(x.producer, data_source) self.assertEqual(x.producer_args, {}) - - From ce5cb0777715ba3759071e92271f92c46a99525b Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 5 Feb 2020 16:07:31 -0800 Subject: [PATCH 06/70] passing core tests ? Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 4 +- nemo/backends/pytorch/common/zero_data.py | 8 +- nemo/core/neural_types/axes.py | 2 +- nemo/core/neural_types/elements.py | 21 +- nemo/core/neural_types/neural_type.py | 85 ++++-- requirements/requirements_test.txt | 1 + tests/{ => core}/test_actions_api.py | 0 tests/core/test_deploy_export.py | 149 ++++++++++ tests/{ => core}/test_deprecated.py | 0 tests/{ => core}/test_infer.py | 18 +- tests/{ => core}/test_neural_factory.py | 0 .../test_neural_modules_initialization.py | 0 .../{ => core}/test_neural_modules_pytorch.py | 13 +- tests/core/test_neural_types.py | 67 ++++- tests/{ => core}/test_policies.py | 0 tests/test_deploy_export.py | 149 ---------- tests/test_neural_types.py | 258 ------------------ tests/test_tutorials_pytorch.py | 29 -- 18 files changed, 322 insertions(+), 482 deletions(-) rename tests/{ => core}/test_actions_api.py (100%) create mode 100644 tests/core/test_deploy_export.py rename tests/{ => core}/test_deprecated.py (100%) rename tests/{ => core}/test_infer.py (81%) rename tests/{ => core}/test_neural_factory.py (100%) rename tests/{ => core}/test_neural_modules_initialization.py (100%) rename tests/{ => core}/test_neural_modules_pytorch.py (90%) rename tests/{ => core}/test_policies.py (100%) delete mode 100644 tests/test_deploy_export.py delete mode 100644 tests/test_neural_types.py delete mode 100644 tests/test_tutorials_pytorch.py diff --git a/Jenkinsfile b/Jenkinsfile index d0d2b0eaa5b1..7c2ee564dac4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,9 +24,9 @@ pipeline { sh 'python setup.py style' } } - stage('Unittests general') { + stage('Unittests Core') { steps { - sh './reinstall.sh && python -m unittest tests/*.py' + sh './reinstall.sh && python -m unittest tests/core/*.py' } } stage('Unittests ASR') { diff --git a/nemo/backends/pytorch/common/zero_data.py b/nemo/backends/pytorch/common/zero_data.py index 0c7b14fe1a11..18f366c46140 100644 --- a/nemo/backends/pytorch/common/zero_data.py +++ b/nemo/backends/pytorch/common/zero_data.py @@ -18,11 +18,11 @@ def neuralType2TensorShape(neural_type: NeuralType, default_dim=32, skip_batch_a torch.Size """ dims = [] - for axis_ind, axis_type in neural_type.axis2type.items(): - if axis_type._semantics == BatchTag and skip_batch_axis: + for axis in neural_type.axes: + if axis.kind == AxisKind.Batch and skip_batch_axis: continue - if axis_type.dim is not None: - dims.append(axis_type.dim) + if axis.size is not None: + dims.append(axis.size) else: dims.append(default_dim) return torch.Size(dims) diff --git a/nemo/core/neural_types/axes.py b/nemo/core/neural_types/axes.py index 5efba5d20ca7..acb9a27646f2 100644 --- a/nemo/core/neural_types/axes.py +++ b/nemo/core/neural_types/axes.py @@ -69,7 +69,7 @@ class AxisType(object): is_list (bool, default=False): """ - def __init__(self, kind: AxisKindAbstract, size: Optional[int], is_list=False): + def __init__(self, kind: AxisKindAbstract, size: Optional[int] = None, is_list=False): if size is not None and is_list: raise ValueError("The axis can't be list and have a fixed size") self.kind = kind diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index f4c4d12445b4..37f35867a159 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -28,6 +28,9 @@ 'LabelsType', 'LossType', 'RegressionValuesType', + 'PredictionsType', + 'LogprobsType', + 'LengthsType', ] import abc from abc import ABC, abstractmethod @@ -55,7 +58,6 @@ def fields(self) -> Optional[Tuple]: def compare(self, second) -> NeuralTypeComparisonResult: # First, check general compatibility - result = NeuralTypeComparisonResult.SAME first_t = type(self) second_t = type(second) @@ -108,11 +110,21 @@ def __str__(self): return "neural type representing logits" +class LogprobsType(ElementType): + def __str__(self): + return "neural type representing log probabilities" + + class LabelsType(ElementType): def __str__(self): return "neural type representing labels" +class LengthsType(ElementType): + def __str__(self): + return "neural type representing lengths of something" + + class LossType(ElementType): def __str__(self): return "neural type representing loss value" @@ -151,6 +163,11 @@ def __str__(self): return "mfcc spectorgram type" -class RegressionValuesType(ElementType): +class PredictionsType(ElementType): + def __str__(self): + return "predictions values type" + + +class RegressionValuesType(PredictionsType): def __str__(self): return "regression values type" diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index a2df777c9296..9cb7513963e4 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -44,7 +44,7 @@ def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple ) self.elements_type = elements_type if axes is not None: - self.__check_sanity(axes) + NeuralType.__check_sanity(axes) axes_list = [] for axis in axes: if isinstance(axis, str): @@ -63,9 +63,7 @@ def compare(self, second) -> NeuralTypeComparisonResult: axes_a = self.axes axes_b = second.axes - kinds_a = dict() - kinds_b = dict() - + # "Big void" type if isinstance(self.elements_type, VoidType) and self.axes is None: return NeuralTypeComparisonResult.SAME @@ -75,28 +73,29 @@ def compare(self, second) -> NeuralTypeComparisonResult: else: return NeuralTypeComparisonResult.INCOMPATIBLE - dimensions_pass = True - for axis_a, axis_b in zip(axes_a, axes_b): - kinds_a[axis_a.kind] = axis_a.size - kinds_b[axis_b.kind] = axis_b.size - if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list: - dimensions_pass = False - - if kinds_a.keys() != kinds_b.keys(): - return NeuralTypeComparisonResult.INCOMPATIBLE - for kind, size in kinds_a.items(): - if size != kinds_b[kind]: - return NeuralTypeComparisonResult.DIM_INCOMPATIBLE - + dimensions_pass = NeuralType.__compare_axes(axes_a, axes_b) element_comparison_result = self.elements_type.compare(second.elements_type) - if dimensions_pass: + + # SAME DIMS + if dimensions_pass == 0: return element_comparison_result - elif element_comparison_result == NeuralTypeComparisonResult.SAME: - return NeuralTypeComparisonResult.TRANSPOSE_SAME + # TRANSPOSE_SAME DIMS + elif dimensions_pass == 1: + if element_comparison_result == NeuralTypeComparisonResult.SAME: + return NeuralTypeComparisonResult.TRANSPOSE_SAME + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + # DIM_INCOMPATIBLE DIMS + elif dimensions_pass == 2: + if element_comparison_result == NeuralTypeComparisonResult.SAME: + return NeuralTypeComparisonResult.DIM_INCOMPATIBLE + else: + return NeuralTypeComparisonResult.INCOMPATIBLE else: return NeuralTypeComparisonResult.INCOMPATIBLE - def __check_sanity(self, axes): + @staticmethod + def __check_sanity(axes): # check that list come before any tensor dimension are_strings = True for axis in axes: @@ -119,6 +118,50 @@ def __check_sanity(self, axes): "You have list dimension after Tensor dimension. All list dimensions must preceed Tensor dimensions" ) + @staticmethod + def __compare_axes(axes_a, axes_b) -> int: + """ + Compares axes_a and axes_b + Args: + axes_a: first axes tuple + axes_b: second axes tuple + + Returns: + 0 - if they are exactly the same + 1 - if they are "TRANSPOSE_SAME" + 2 - if the are "DIM_INCOMPATIBLE" + 3 - if they are different + """ + if axes_a is None and axes_b is None: + return 0 + elif axes_a is None and axes_b is not None: + return 3 + elif axes_a is not None and axes_b is None: + return 3 + elif len(axes_a) != len(axes_b): + return 3 + # After these ifs we know that len(axes_a) == len(axes_b) + + same = True + kinds_a = dict() + kinds_b = dict() + for axis_a, axis_b in zip(axes_a, axes_b): + kinds_a[axis_a.kind] = axis_a.size + kinds_b[axis_b.kind] = axis_b.size + if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list or axis_a.size != axis_b.size: + same = False + if same: + return 0 + else: + # can be TRANSPOSE_SAME, DIM_INCOMPATIBLE + if kinds_a.keys() == kinds_b.keys(): + for key, value in kinds_a.items(): + if kinds_b[key] != value: + return 2 + return 1 + else: + return 3 + class NmTensor(NeuralType): """Class representing data which flows between NeuralModules' ports. diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index 493b8268cfd1..544127fca734 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -4,3 +4,4 @@ pytest-runner black isort[requirements] wrapt +onnxruntime diff --git a/tests/test_actions_api.py b/tests/core/test_actions_api.py similarity index 100% rename from tests/test_actions_api.py rename to tests/core/test_actions_api.py diff --git a/tests/core/test_deploy_export.py b/tests/core/test_deploy_export.py new file mode 100644 index 000000000000..6ef415c6c8cf --- /dev/null +++ b/tests/core/test_deploy_export.py @@ -0,0 +1,149 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2019 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# import os +# from pathlib import Path +# +# # git clone git@github.com:microsoft/onnxruntime.git +# # cd onnxruntime +# # ./build.sh --update --build --config RelWithDebInfo --build_shared_lib --parallel --use_cuda \ +# # --cudnn_home /usr/lib/x86_64-linux-gnu --cuda_home /usr/local/cuda --enable_pybind --build_wheel +# # pip install --upgrade ./build/Linux/RelWithDebInfo/dist/onnxruntime_gpu-1.1.0-cp37-cp37m-linux_x86_64.whl +# import onnxruntime as ort +# import torch +# from ruamel.yaml import YAML +# +# import nemo +# import nemo.collections.asr as nemo_asr +# import nemo.collections.nlp as nemo_nlp +# import nemo.collections.nlp.nm.trainables.common.token_classification_nm +# from tests.common_setup import NeMoUnitTest +# +# +# class TestDeployExport(NeMoUnitTest): +# def setUp(self): +# """ Setups neural factory so it will use GPU instead of CPU. """ +# NeMoUnitTest.setUp(self) +# +# # Perform computations on GPU. +# self.nf._placement = nemo.core.DeviceType.GPU +# +# def __test_export_route(self, module, out_name, mode, input_example=None): +# out = Path(out_name) +# if out.exists(): +# os.remove(out) +# +# self.nf.deployment_export(module=module, output=out_name, input_example=input_example, d_format=mode) +# +# self.assertTrue(out.exists()) +# if mode == nemo.core.DeploymentFormat.ONNX: +# if isinstance(input_example, tuple): +# outputs_fwd = module.forward(*input_example) +# else: +# outputs_fwd = module.forward(input_example) +# sess_options = ort.SessionOptions() +# sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED +# ort_session = ort.InferenceSession(out_name, sess_options) +# inputs = dict() +# input_names = list(module.input_ports) +# for i in range(len(input_names)): +# input_name = ( +# "encoded_lengths" +# if type(module).__name__ == "JasperEncoder" and input_names[i] == "length" +# else input_names[i] +# ) +# inputs[input_name] = ( +# input_example[i].cpu().numpy() if isinstance(input_example, tuple) else input_example.cpu().numpy() +# ) +# outputs_ort = ort_session.run(None, inputs) +# outputs_ort = torch.from_numpy(outputs_ort[0]).cuda() +# self.assertLess( +# (outputs_ort - (outputs_fwd[0] if isinstance(outputs_fwd, tuple) else outputs_fwd)).norm(p=2), 5.0e-4 +# ) +# if out.exists(): +# os.remove(out) +# +# def test_simple_module_export(self): +# simplest_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# self.__test_export_route( +# module=simplest_module, +# out_name="simple.pt", +# mode=nemo.core.DeploymentFormat.TORCHSCRIPT, +# input_example=None, +# ) +# +# def test_TokenClassifier_module_export(self): +# t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( +# hidden_size=512, num_classes=16, use_transformer_pretrained=False +# ) +# self.__test_export_route( +# module=t_class, +# out_name="t_class.pt", +# mode=nemo.core.DeploymentFormat.TORCHSCRIPT, +# input_example=torch.randn(16, 16, 512).cuda(), +# ) +# +# def test_TokenClassifier_module_onnx_export(self): +# t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( +# hidden_size=512, num_classes=16, use_transformer_pretrained=False +# ) +# self.__test_export_route( +# module=t_class, +# out_name="t_class.onnx", +# mode=nemo.core.DeploymentFormat.ONNX, +# input_example=torch.randn(16, 16, 512).cuda(), +# ) +# +# def test_jasper_decoder_export_ts(self): +# j_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=33) +# self.__test_export_route( +# module=j_decoder, out_name="j_decoder.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=None +# ) +# +# def test_hf_bert_ts(self): +# bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") +# input_example = ( +# torch.randint(low=0, high=16, size=(2, 16)).cuda(), +# torch.randint(low=0, high=1, size=(2, 16)).cuda(), +# torch.randint(low=0, high=1, size=(2, 16)).cuda(), +# ) +# self.__test_export_route( +# module=bert, out_name="bert.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=input_example +# ) +# +# def test_hf_bert_pt(self): +# bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") +# self.__test_export_route(module=bert, out_name="bert.pt", mode=nemo.core.DeploymentFormat.PYTORCH) +# +# def test_jasper_encoder_to_onnx(self): +# with open("tests/data/jasper_smaller.yaml") as file: +# yaml = YAML(typ="safe") +# jasper_model_definition = yaml.load(file) +# +# jasper_encoder = nemo_asr.JasperEncoder( +# conv_mask=False, +# feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], +# **jasper_model_definition['JasperEncoder'] +# ) +# +# self.__test_export_route( +# module=jasper_encoder, +# out_name="jasper_encoder.onnx", +# mode=nemo.core.DeploymentFormat.ONNX, +# input_example=(torch.randn(16, 64, 256).cuda(), torch.randn(256).cuda()), +# ) diff --git a/tests/test_deprecated.py b/tests/core/test_deprecated.py similarity index 100% rename from tests/test_deprecated.py rename to tests/core/test_deprecated.py diff --git a/tests/test_infer.py b/tests/core/test_infer.py similarity index 81% rename from tests/test_infer.py rename to tests/core/test_infer.py index 05cec60c6fb9..e9611ea43967 100644 --- a/tests/test_infer.py +++ b/tests/core/test_infer.py @@ -30,11 +30,13 @@ def __init__(self): @property def input_ports(self): - return {"mod_in": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + # return {"mod_in": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + return {"mod_in": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} @property def output_ports(self): - return {"mod_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + # return {"mod_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + return {"mod_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} def forward(self, mod_in): return mod_in + 10 @@ -46,11 +48,11 @@ def __init__(self): @property def input_ports(self): - return {"mod_in": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + return {"mod_in": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} @property def output_ports(self): - return {"mod_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} + return {"mod_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} def forward(self, mod_in): return mod_in - 10 @@ -66,7 +68,9 @@ def test_infer_caching(self): size=1, dtype=torch.FloatTensor, batch_size=1, - output_ports={"dl_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})}, + output_ports={ + "dl_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1))) + }, ) addten = AddsTen() minusten = SubtractsTen() @@ -93,7 +97,9 @@ def test_infer_errors(self): size=1, dtype=torch.FloatTensor, batch_size=1, - output_ports={"dl_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})}, + output_ports={ + "dl_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1))) + }, ) addten = AddsTen() minusten = SubtractsTen() diff --git a/tests/test_neural_factory.py b/tests/core/test_neural_factory.py similarity index 100% rename from tests/test_neural_factory.py rename to tests/core/test_neural_factory.py diff --git a/tests/test_neural_modules_initialization.py b/tests/core/test_neural_modules_initialization.py similarity index 100% rename from tests/test_neural_modules_initialization.py rename to tests/core/test_neural_modules_initialization.py diff --git a/tests/test_neural_modules_pytorch.py b/tests/core/test_neural_modules_pytorch.py similarity index 90% rename from tests/test_neural_modules_pytorch.py rename to tests/core/test_neural_modules_pytorch.py index 13ff0226262b..236844031e25 100644 --- a/tests/test_neural_modules_pytorch.py +++ b/tests/core/test_neural_modules_pytorch.py @@ -17,10 +17,13 @@ # limitations under the License. # ============================================================================= +# TODO: These test look bad/useless - redo + import unittest import nemo from nemo.backends.pytorch.nm import TrainableNM +from nemo.core.neural_types import ChannelType, NeuralType from tests.common_setup import NeMoUnitTest @@ -67,15 +70,7 @@ def test_constructor_TaylorNet(self): def test_call_TaylorNet(self): x_tg = nemo.core.neural_modules.NmTensor( - producer=None, - producer_args=None, - name=None, - ntype=nemo.core.neural_types.NeuralType( - { - 0: nemo.core.neural_types.AxisType(nemo.core.neural_types.BatchTag), - 1: nemo.core.neural_types.AxisType(nemo.core.neural_types.ChannelTag), - } - ), + producer=None, producer_args=None, name=None, ntype=NeuralType(ChannelType(), ('B', 'D')) ) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index 51a1c9b18044..537813b76f07 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= - +import nemo from nemo.core.neural_types import ( AcousticEncodedRepresentation, AudioSignal, @@ -24,9 +24,11 @@ ChannelType, MelSpectrogramType, MFCCSpectrogramType, + NeuralPortNmTensorMismatchError, NeuralType, NeuralTypeComparisonResult, SpectrogramType, + VoidType, ) from tests.common_setup import NeMoUnitTest @@ -102,3 +104,66 @@ def test_list_of_lists(self): ) # TODO: should this be incompatible instead??? self.assertEqual(T1.compare(T2), NeuralTypeComparisonResult.TRANSPOSE_SAME) + + def test_void(self): + btc_spctr = NeuralType(SpectrogramType(), ('B', 'T', 'C')) + btc_spct_bad = NeuralType(SpectrogramType(), ('B', 'T')) + btc_void = NeuralType(VoidType(), ('B', 'T', 'C')) + self.assertEqual(btc_void.compare(btc_spctr), NeuralTypeComparisonResult.SAME) + self.assertEqual(btc_spctr.compare(btc_void), NeuralTypeComparisonResult.INCOMPATIBLE) + self.assertEqual(btc_void.compare(btc_spct_bad), NeuralTypeComparisonResult.INCOMPATIBLE) + + def test_big_void(self): + big_void_1 = NeuralType(VoidType()) + big_void_2 = NeuralType() + + btc_spctr = NeuralType(SpectrogramType(), ('B', 'T', 'C')) + btc_spct_bad = NeuralType(SpectrogramType(), ('B', 'T')) + t1 = NeuralType( + elements_type=ChannelType(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + ) + t2 = NeuralType( + elements_type=ChannelType(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + ) + + self.assertEqual(big_void_1.compare(btc_spctr), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_1.compare(btc_spct_bad), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_1.compare(t1), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_1.compare(t2), NeuralTypeComparisonResult.SAME) + + self.assertEqual(big_void_2.compare(btc_spctr), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_2.compare(btc_spct_bad), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_2.compare(t1), NeuralTypeComparisonResult.SAME) + self.assertEqual(big_void_2.compare(t2), NeuralTypeComparisonResult.SAME) + + def test_dag(self): + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) + trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) + loss = nemo.backends.pytorch.tutorials.MSELoss() + x, y = data_source() + y_pred = trainable_module(x=x) + _ = loss(predictions=y_pred, target=y) + + def wrong(): + data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=10000, batch_size=128) + trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) + loss = nemo.backends.pytorch.tutorials.MSELoss() + x, y = data_source() + loss_tensor = loss(predictions=x, target=x) + _ = trainable_module(x=loss_tensor) + + self.assertRaises(NeuralPortNmTensorMismatchError, wrong) diff --git a/tests/test_policies.py b/tests/core/test_policies.py similarity index 100% rename from tests/test_policies.py rename to tests/core/test_policies.py diff --git a/tests/test_deploy_export.py b/tests/test_deploy_export.py deleted file mode 100644 index be6a1a39573c..000000000000 --- a/tests/test_deploy_export.py +++ /dev/null @@ -1,149 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright 2019 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import os -from pathlib import Path - -# git clone git@github.com:microsoft/onnxruntime.git -# cd onnxruntime -# ./build.sh --update --build --config RelWithDebInfo --build_shared_lib --parallel --use_cuda \ -# --cudnn_home /usr/lib/x86_64-linux-gnu --cuda_home /usr/local/cuda --enable_pybind --build_wheel -# pip install --upgrade ./build/Linux/RelWithDebInfo/dist/onnxruntime_gpu-1.1.0-cp37-cp37m-linux_x86_64.whl -import onnxruntime as ort -import torch -from ruamel.yaml import YAML - -import nemo -import nemo.collections.asr as nemo_asr -import nemo.collections.nlp as nemo_nlp -import nemo.collections.nlp.nm.trainables.common.token_classification_nm -from tests.common_setup import NeMoUnitTest - - -class TestDeployExport(NeMoUnitTest): - def setUp(self): - """ Setups neural factory so it will use GPU instead of CPU. """ - NeMoUnitTest.setUp(self) - - # Perform computations on GPU. - self.nf._placement = nemo.core.DeviceType.GPU - - def __test_export_route(self, module, out_name, mode, input_example=None): - out = Path(out_name) - if out.exists(): - os.remove(out) - - self.nf.deployment_export(module=module, output=out_name, input_example=input_example, d_format=mode) - - self.assertTrue(out.exists()) - if mode == nemo.core.DeploymentFormat.ONNX: - if isinstance(input_example, tuple): - outputs_fwd = module.forward(*input_example) - else: - outputs_fwd = module.forward(input_example) - sess_options = ort.SessionOptions() - sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED - ort_session = ort.InferenceSession(out_name, sess_options) - inputs = dict() - input_names = list(module.input_ports) - for i in range(len(input_names)): - input_name = ( - "encoded_lengths" - if type(module).__name__ == "JasperEncoder" and input_names[i] == "length" - else input_names[i] - ) - inputs[input_name] = ( - input_example[i].cpu().numpy() if isinstance(input_example, tuple) else input_example.cpu().numpy() - ) - outputs_ort = ort_session.run(None, inputs) - outputs_ort = torch.from_numpy(outputs_ort[0]).cuda() - self.assertLess( - (outputs_ort - (outputs_fwd[0] if isinstance(outputs_fwd, tuple) else outputs_fwd)).norm(p=2), 5.0e-4 - ) - if out.exists(): - os.remove(out) - - def test_simple_module_export(self): - simplest_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - self.__test_export_route( - module=simplest_module, - out_name="simple.pt", - mode=nemo.core.DeploymentFormat.TORCHSCRIPT, - input_example=None, - ) - - def test_TokenClassifier_module_export(self): - t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( - hidden_size=512, num_classes=16, use_transformer_pretrained=False - ) - self.__test_export_route( - module=t_class, - out_name="t_class.pt", - mode=nemo.core.DeploymentFormat.TORCHSCRIPT, - input_example=torch.randn(16, 16, 512).cuda(), - ) - - def test_TokenClassifier_module_onnx_export(self): - t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( - hidden_size=512, num_classes=16, use_transformer_pretrained=False - ) - self.__test_export_route( - module=t_class, - out_name="t_class.onnx", - mode=nemo.core.DeploymentFormat.ONNX, - input_example=torch.randn(16, 16, 512).cuda(), - ) - - def test_jasper_decoder_export_ts(self): - j_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=33) - self.__test_export_route( - module=j_decoder, out_name="j_decoder.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=None - ) - - def test_hf_bert_ts(self): - bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") - input_example = ( - torch.randint(low=0, high=16, size=(2, 16)).cuda(), - torch.randint(low=0, high=1, size=(2, 16)).cuda(), - torch.randint(low=0, high=1, size=(2, 16)).cuda(), - ) - self.__test_export_route( - module=bert, out_name="bert.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=input_example - ) - - def test_hf_bert_pt(self): - bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") - self.__test_export_route(module=bert, out_name="bert.pt", mode=nemo.core.DeploymentFormat.PYTORCH) - - def test_jasper_encoder_to_onnx(self): - with open("tests/data/jasper_smaller.yaml") as file: - yaml = YAML(typ="safe") - jasper_model_definition = yaml.load(file) - - jasper_encoder = nemo_asr.JasperEncoder( - conv_mask=False, - feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], - **jasper_model_definition['JasperEncoder'] - ) - - self.__test_export_route( - module=jasper_encoder, - out_name="jasper_encoder.onnx", - mode=nemo.core.DeploymentFormat.ONNX, - input_example=(torch.randn(16, 64, 256).cuda(), torch.randn(256).cuda()), - ) diff --git a/tests/test_neural_types.py b/tests/test_neural_types.py deleted file mode 100644 index c2741ca3d7c6..000000000000 --- a/tests/test_neural_types.py +++ /dev/null @@ -1,258 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright 2019 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import tarfile - -from ruamel.yaml import YAML - -import nemo.collections.asr as nemo_asr -from nemo import logging -from nemo.core import * -from tests.common_setup import NeMoUnitTest - - -class TestNeuralTypes(NeMoUnitTest): - manifest_filepath = "tests/data/asr/an4_train.json" - yaml = YAML(typ="safe") - - def setUp(self) -> None: - super().setUp() - data_folder = "tests/data/" - logging.info("Looking up for test ASR data") - if not os.path.exists(data_folder + "asr"): - logging.info("Extracting ASR data to: {0}".format(data_folder + "asr")) - tar = tarfile.open("tests/data/asr.tar.gz", "r:gz") - tar.extractall(path=data_folder) - tar.close() - else: - logging.info("ASR data found in: {0}".format(data_folder + "asr")) - - def test_same(self): - btc = NeuralType(axis2type={0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - btc2 = NeuralType(axis2type={0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - self.assertEqual(btc2.compare(btc), NeuralTypeComparisonResult.SAME) - - def test_transpose_same(self): - btc = NeuralType(axis2type={0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - tbc = NeuralType(axis2type={1: AxisType(BatchTag), 0: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - - self.assertEqual(btc.compare(tbc), NeuralTypeComparisonResult.TRANSPOSE_SAME) - self.assertEqual(tbc.compare(btc), NeuralTypeComparisonResult.TRANSPOSE_SAME) - - def test_dim_incompatible(self): - nchw1 = NeuralType( - axis2type={ - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - } - ) - nchw2 = NeuralType( - axis2type={ - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 256), - 3: AxisType(WidthTag, 256), - } - ) - self.assertEqual(nchw1.compare(nchw2), NeuralTypeComparisonResult.DIM_INCOMPATIBLE) - - def test_rank_incompatible(self): - btc = NeuralType(axis2type={0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - nchw = NeuralType( - axis2type={0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(HeightTag), 3: AxisType(WidthTag)} - ) - self.assertEqual(nchw.compare(btc), NeuralTypeComparisonResult.INCOMPATIBLE) - - def test_axis_type(self): - ax1 = AxisType(BatchTag) - ax2 = AxisType(TimeTag) - ax3 = AxisType(ProcessedTimeTag) - self.assertEqual(ax1.compare_to(ax2), NeuralTypeComparisonResult.INCOMPATIBLE) - self.assertEqual(ax3.compare_to(ax2), NeuralTypeComparisonResult.LESS) - self.assertEqual(ax2.compare_to(ax3), NeuralTypeComparisonResult.GREATER) - self.assertEqual(ax2.compare_to(AxisType(TimeTag)), NeuralTypeComparisonResult.SAME) - - def test_semantic_incompatible(self): - nchw = NeuralType( - axis2type={0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(HeightTag), 3: AxisType(WidthTag)} - ) - badd = NeuralType( - axis2type={0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag), 3: AxisType(WidthTag)} - ) - self.assertEqual(nchw.compare(badd), NeuralTypeComparisonResult.INCOMPATIBLE) - self.assertEqual(badd.compare(nchw), NeuralTypeComparisonResult.INCOMPATIBLE) - - def test_root(self): - root = NeuralType({}) - non_tensor = NeuralType(None) - btc = NeuralType(axis2type={0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}) - nchw = NeuralType( - axis2type={0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(HeightTag), 3: AxisType(WidthTag)} - ) - self.assertEqual(root.compare(btc), NeuralTypeComparisonResult.SAME) - self.assertEqual(root.compare(nchw), NeuralTypeComparisonResult.SAME) - self.assertEqual(root.compare(non_tensor), NeuralTypeComparisonResult.SAME) - - self.assertEqual(non_tensor.compare(root), NeuralTypeComparisonResult.INCOMPATIBLE) - self.assertEqual(btc.compare(root), NeuralTypeComparisonResult.INCOMPATIBLE) - self.assertEqual(nchw.compare(root), NeuralTypeComparisonResult.INCOMPATIBLE) - - def test_combiner_type_infer(self): - combiner = nemo.backends.pytorch.common.SimpleCombiner(mode="add") - x_tg = nemo.core.NmTensor( - producer=None, producer_args=None, name=None, ntype=NeuralType({0: AxisType(BatchTag)}) - ) - y_tg = nemo.core.NmTensor( - producer=None, producer_args=None, name=None, ntype=NeuralType({0: AxisType(BatchTag)}) - ) - res = combiner(x1=y_tg, x2=x_tg) - self.assertEqual(res.compare(x_tg), NeuralTypeComparisonResult.SAME) - self.assertEqual(res.compare(y_tg), NeuralTypeComparisonResult.SAME) - self.assertEqual(x_tg.compare(res), NeuralTypeComparisonResult.SAME) - self.assertEqual(y_tg.compare(res), NeuralTypeComparisonResult.SAME) - - combiner1 = nemo.backends.pytorch.common.SimpleCombiner(mode="add") - x_tg1 = NmTensor( - producer=None, - producer_args=None, - name=None, - ntype=NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - ) - y_tg1 = NmTensor( - producer=None, - producer_args=None, - name=None, - ntype=NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - ) - res1 = combiner1(x1=y_tg1, x2=x_tg1) - self.assertEqual(res1.compare(x_tg1), NeuralTypeComparisonResult.SAME) - self.assertEqual(res1.compare(y_tg1), NeuralTypeComparisonResult.SAME) - self.assertEqual(x_tg1.compare(res1), NeuralTypeComparisonResult.SAME) - self.assertEqual(y_tg1.compare(res1), NeuralTypeComparisonResult.SAME) - - def test_optional_input_no_input(self): - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=100, batch_size=128) - trainable_module = nemo.backends.pytorch.tutorials.TaylorNetO(dim=4) - loss = nemo.backends.pytorch.tutorials.MSELoss() - x, y = data_source() - y_pred = trainable_module(x=x) - loss_tensor = loss(predictions=y_pred, target=y) - - optimizer = nemo.backends.pytorch.actions.PtActions() - optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1} - ) - - def test_optional_input_no_with_input(self): - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=100, batch_size=128) - trainable_module = nemo.backends.pytorch.tutorials.TaylorNetO(dim=4) - loss = nemo.backends.pytorch.tutorials.MSELoss() - x, y = data_source() - y_pred = trainable_module(x=x, o=x) - loss_tensor = loss(predictions=y_pred, target=y) - optimizer = nemo.backends.pytorch.actions.PtActions() - optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1} - ) - - def test_optional_input_no_with_wrong_input(self): - def wrong_fn(): - data_source = nemo.backends.pytorch.tutorials.RealFunctionDataLayer(n=100, batch_size=128) - trainable_module = nemo.backends.pytorch.tutorials.TaylorNetO(dim=4) - loss = nemo.backends.pytorch.tutorials.MSELoss() - x, y = data_source() - wrong_optional = NmTensor( - producer=None, - producer_args=None, - name=None, - ntype=NeuralType({0: AxisType(ChannelTag), 1: AxisType(BatchTag)}), - ) - y_pred = trainable_module(x=x, o=wrong_optional) - loss_tensor = loss(predictions=y_pred, target=y) - optimizer = nemo.backends.pytorch.actions.PtActions() - optimizer.train( - tensors_to_optimize=[loss_tensor], optimizer="sgd", optimization_params={"lr": 0.0003, "num_epochs": 1} - ) - - self.assertRaises(NeuralPortNmTensorMismatchError, wrong_fn) - - def test_simple_dags(self): - # module instantiation - with open("tests/data/jasper_smaller.yaml") as file: - jasper_model_definition = self.yaml.load(file) - labels = jasper_model_definition['labels'] - - data_layer = nemo_asr.AudioToTextDataLayer( - manifest_filepath=self.manifest_filepath, labels=labels, batch_size=4 - ) - data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor( - **jasper_model_definition['AudioToMelSpectrogramPreprocessor'] - ) - jasper_encoder = nemo_asr.JasperEncoder( - feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], - **jasper_model_definition['JasperEncoder'], - ) - jasper_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(labels)) - ctc_loss = nemo_asr.CTCLossNM(num_classes=len(labels)) - greedy_decoder = nemo_asr.GreedyCTCDecoder() - - # DAG definition - (audio_signal, audio_signal_len, transcript, transcript_len) = data_layer() - processed_signal, processed_signal_len = data_preprocessor(input_signal=audio_signal, length=audio_signal_len) - - spec_augment = nemo_asr.SpectrogramAugmentation(rect_masks=5) - aug_signal = spec_augment(input_spec=processed_signal) - - encoded, encoded_len = jasper_encoder(audio_signal=aug_signal, length=processed_signal_len) - log_probs = jasper_decoder(encoder_output=encoded) - predictions = greedy_decoder(log_probs=log_probs) - loss = ctc_loss( - log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len - ) - - def wrong(): - with open("tests/data/jasper_smaller.yaml") as file: - jasper_config = self.yaml.load(file) - labels = jasper_config['labels'] - - data_layer = nemo_asr.AudioToTextDataLayer( - manifest_filepath=self.manifest_filepath, labels=labels, batch_size=4 - ) - data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor( - **jasper_config['AudioToMelSpectrogramPreprocessor'] - ) - jasper_encoder = nemo_asr.JasperEncoder( - feat_in=jasper_config['AudioToMelSpectrogramPreprocessor']['features'], - **jasper_config['JasperEncoder'], - ) - jasper_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(labels)) - # DAG definition - (audio_signal, audio_signal_len, transcript, transcript_len) = data_layer() - processed_signal, processed_signal_len = data_preprocessor( - input_signal=audio_signal, length=audio_signal_len - ) - - spec_augment = nemo_asr.SpectrogramAugmentation(rect_masks=5) - aug_signal = spec_augment(input_spec=processed_signal) - - encoded, encoded_len = jasper_encoder(audio_signal=aug_signal, length=processed_signal_len) - log_probs = jasper_decoder(encoder_output=processed_signal) - - self.assertRaises(NeuralPortNmTensorMismatchError, wrong) diff --git a/tests/test_tutorials_pytorch.py b/tests/test_tutorials_pytorch.py deleted file mode 100644 index 183fd67e1d1b..000000000000 --- a/tests/test_tutorials_pytorch.py +++ /dev/null @@ -1,29 +0,0 @@ -# # ! /usr/bin/python -# # -*- coding: utf-8 -*- -# -# # Copyright 2019 NVIDIA. All Rights Reserved. -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License. -# # ============================================================================= -# -# from .common_setup import NeMoUnitTest -# from nemo.backends.pytorch.tutorials.chatbot.data import loadPrepareData -# -# -# class TestPytorchChatBotTutorial(NeMoUnitTest): -# def test_simple_train(self): -# datafile = "tests/data/dialog_sample.txt" -# logging.info(datafile) -# voc, pairs = loadPrepareData("cornell", datafile=datafile) -# self.assertEqual(voc.name, 'cornell') -# self.assertEqual(voc.num_words, 675) From ca9c370cc3935cdc9f0ed382a35895b82ac1280d Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 5 Feb 2020 18:20:09 -0800 Subject: [PATCH 07/70] asr and core tests are passing Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 2 +- nemo/backends/pytorch/common/other.py | 15 +- nemo/backends/pytorch/common/rnn.py | 22 +- nemo/collections/asr/__init__.py | 2 +- nemo/collections/asr/audio_preprocessing.py | 210 +++++---------- nemo/collections/asr/beam_search_decoder.py | 18 +- nemo/collections/asr/data_layer.py | 89 +++---- nemo/collections/asr/greedy_ctc_decoder.py | 20 +- nemo/collections/asr/jasper.py | 76 ++---- nemo/collections/asr/las/misc.py | 15 +- nemo/collections/asr/losses.py | 35 +-- nemo/core/neural_types/neural_type.py | 3 +- tests/asr/test_asr.py | 10 +- tests/asr/test_weight_share.py | 271 -------------------- tests/asr/test_zeroDS.py | 50 ++-- tests/core/test_neural_types.py | 7 + tests/core/test_weight_share.py | 220 ++++++++++++++++ 17 files changed, 433 insertions(+), 632 deletions(-) delete mode 100644 tests/asr/test_weight_share.py create mode 100644 tests/core/test_weight_share.py diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index 633eee772b66..4cacb1853620 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -45,7 +45,7 @@ def output_ports(self): NeuralType(None) """ - return {"loss": NeuralType(elements_type=LossType)} + return {"loss": NeuralType(LossType())} def __init__( self, pad_id=0, smoothing_coef=0.0, sample_wise=False, aux_ctc=False, ctc_initial_coef=0.1, ctc_blank_id=None diff --git a/nemo/backends/pytorch/common/other.py b/nemo/backends/pytorch/common/other.py index b5ba4be753c0..9358f586d387 100644 --- a/nemo/backends/pytorch/common/other.py +++ b/nemo/backends/pytorch/common/other.py @@ -34,7 +34,11 @@ class SimpleCombiner(TrainableNM): def input_ports(self): """Returns definitions of module input ports. """ - return {"x1": NeuralType(VoidType()), "x2": NeuralType(VoidType())} + if self._input_ports is None: + return {"x1": NeuralType(VoidType()), "x2": NeuralType(VoidType())} + else: + return self._input_ports + @property def output_ports(self): @@ -43,11 +47,16 @@ def output_ports(self): combined: None """ - return {"combined": NeuralType(VoidType())} + if self._output_ports is None: + return {"combined": NeuralType(VoidType())} + else: + return self._output_ports - def __init__(self, mode="add"): + def __init__(self, mode="add", input_ports=None, output_ports=None): super().__init__() self._mode = mode + self._input_ports = input_ports + self._output_ports = output_ports def forward(self, x1, x2): if self._mode == "add" or self._mode == "sum": diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index 4d87a2cca196..7136a569fb23 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -22,7 +22,7 @@ from nemo.backends.pytorch.common.parts import Attention from nemo.backends.pytorch.nm import TrainableNM -from nemo.core import AxisType +from nemo.core import * from nemo.utils.misc import pad_to __all__ = ['DecoderRNN', 'EncoderRNN'] @@ -81,10 +81,12 @@ def input_ports(self): 2: AxisType(ChannelTag) """ return { - 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'encoder_outputs': NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, - ), + # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + 'targets': NeuralType(ChannelType(), ('B', 'T')), + # 'encoder_outputs': NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, + # ), + 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'D'), True) } @property @@ -106,10 +108,12 @@ def output_ports(self): 2: AxisType(TimeTag) """ return { - 'log_probs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), - 'attention_weights': NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}, optional=True, - ), + # 'log_probs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), + 'log_probs': NeuralType(LogprobsType(), ('B', 'T', 'D')), + # 'attention_weights': NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}, optional=True, + # ), + 'attention_weights': NeuralType(ChannelType(), ('B', 'T', 'T'), True) } def __init__( diff --git a/nemo/collections/asr/__init__.py b/nemo/collections/asr/__init__.py index b84913f0ce8d..84c9501c6233 100644 --- a/nemo/collections/asr/__init__.py +++ b/nemo/collections/asr/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019 NVIDIA. All Rights Reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo/collections/asr/audio_preprocessing.py b/nemo/collections/asr/audio_preprocessing.py index 94476839a1f3..d16f9e9afa76 100644 --- a/nemo/collections/asr/audio_preprocessing.py +++ b/nemo/collections/asr/audio_preprocessing.py @@ -1,16 +1,17 @@ -# Copyright (C) NVIDIA CORPORATION. All Rights Reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License.**** +# limitations under the License. +# ============================================================================= """ This file contains neural modules responsible for preprocessing audio data. """ @@ -131,32 +132,24 @@ def input_ports(self): """ return { - "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "length": NeuralType({0: AxisType(BatchTag)}), + # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "length": NeuralType({0: AxisType(BatchTag)}), + "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), + "length": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - processed_signal: - - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(ProcessedTimeTag) - - processed_length: - - 0: AxisType(BatchTag) - """ return { - "processed_signal": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} - ), - "processed_length": NeuralType({0: AxisType(BatchTag)}), + # "processed_signal": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} + # ), + # "processed_length": NeuralType({0: AxisType(BatchTag)}), + + "processed_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "processed_length": NeuralType(LengthsType(), tuple('B')) } def __init__( @@ -170,6 +163,7 @@ def __init__( window="hann", normalized=True, ): + self._sample_rate = sample_rate if not HAVE_TORCHAUDIO: raise ModuleNotFoundError( "torchaudio is not installed but is necessary for " @@ -183,9 +177,9 @@ def __init__( f"{self} received both window_stride and " f"n_window_stride. Only one should be specified." ) if window_size: - n_window_size = int(window_size * sample_rate) + n_window_size = int(window_size * self._sample_rate) if window_stride: - n_window_stride = int(window_stride * sample_rate) + n_window_stride = int(window_stride * self._sample_rate) super().__init__(n_window_size, n_window_stride) @@ -283,19 +277,12 @@ class AudioToMelSpectrogramPreprocessor(AudioPreprocessor): @property def input_ports(self): """Returns definitions of module input ports. - - input_signal: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - length: - 0: AxisType(BatchTag) - """ return { - "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "length": NeuralType({0: AxisType(BatchTag)}), + # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "length": NeuralType({0: AxisType(BatchTag)}), + "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), + "length": NeuralType(LengthsType(), tuple('B')) } @property @@ -316,10 +303,12 @@ def output_ports(self): """ return { - "processed_signal": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} - ), - "processed_length": NeuralType({0: AxisType(BatchTag)}), + # "processed_signal": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} + # ), + # "processed_length": NeuralType({0: AxisType(BatchTag)}), + "processed_signal": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "processed_length": NeuralType(LengthsType(), tuple('B')) } def __init__( @@ -346,6 +335,7 @@ def __init__( pad_value=0, mag_power=2.0, ): + self._sample_rate = sample_rate if window_size and n_window_size: raise ValueError(f"{self} received both window_size and " f"n_window_size. Only one should be specified.") if window_stride and n_window_stride: @@ -353,14 +343,14 @@ def __init__( f"{self} received both window_stride and " f"n_window_stride. Only one should be specified." ) if window_size: - n_window_size = int(window_size * sample_rate) + n_window_size = int(window_size * self._sample_rate) if window_stride: - n_window_stride = int(window_stride * sample_rate) + n_window_stride = int(window_stride * self._sample_rate) super().__init__(n_window_size, n_window_stride) self.featurizer = FilterbankFeatures( - sample_rate=sample_rate, + sample_rate=self._sample_rate, n_window_size=n_window_size, n_window_stride=n_window_stride, window=window, @@ -433,43 +423,26 @@ class AudioToMFCCPreprocessor(AudioPreprocessor): @property def input_ports(self): """Returns definitions of module input ports. - - input_signal: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - length: - 0: AxisType(BatchTag) - """ return { - "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "length": NeuralType({0: AxisType(BatchTag)}), + # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "length": NeuralType({0: AxisType(BatchTag)}), + "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), + "length": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - processed_signal: - - 0: AxisType(BatchTag) - - 1: AxisType(MFCCSignalTag) - - 2: AxisType(ProcessedTimeTag) - - processed_length: - - 0: AxisType(BatchTag) - """ return { - "processed_signal": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MFCCSignalTag), 2: AxisType(ProcessedTimeTag),} - ), - "processed_length": NeuralType({0: AxisType(BatchTag)}), + # "processed_signal": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MFCCSignalTag), 2: AxisType(ProcessedTimeTag),} + # ), + # "processed_length": NeuralType({0: AxisType(BatchTag)}), + "processed_signal": NeuralType(MFCCSpectrogramType(), ('B', 'D', 'T')), + "processed_length": NeuralType(LengthsType(), tuple('B')) + } def __init__( @@ -489,6 +462,7 @@ def __init__( norm='ortho', log=True, ): + self._sample_rate = sample_rate if not HAVE_TORCHAUDIO: raise ModuleNotFoundError( "torchaudio is not installed but is necessary for " @@ -503,9 +477,9 @@ def __init__( ) # Get win_length (n_window_size) and hop_length (n_window_stride) if window_size: - n_window_size = int(window_size * sample_rate) + n_window_size = int(window_size * self._sample_rate) if window_stride: - n_window_stride = int(window_stride * sample_rate) + n_window_stride = int(window_stride * self._sample_rate) super().__init__(n_window_size, n_window_stride) @@ -531,7 +505,7 @@ def __init__( # Use torchaudio's implementation of MFCCs as featurizer self.featurizer = torchaudio.transforms.MFCC( - sample_rate=sample_rate, n_mfcc=n_mfcc, dct_type=dct_type, norm=norm, log_mels=log, melkwargs=mel_kwargs, + sample_rate=self._sample_rate, n_mfcc=n_mfcc, dct_type=dct_type, norm=norm, log_mels=log, melkwargs=mel_kwargs, ) self.featurizer.to(self._device) @@ -575,36 +549,22 @@ class SpectrogramAugmentation(NonTrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_spec: - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(TimeTag) - """ return { - "input_spec": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(TimeTag),}) + # "input_spec": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType( + # TimeTag),}) + "input_spec": NeuralType(SpectrogramType(), ('B', 'D', 'T')) } @property def output_ports(self): """Returns definitions of module output ports. - - augmented_spec: - - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(ProcessedTimeTag) - """ return { - "augmented_spec": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} - ) + # "augmented_spec": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} + # ) + "augmented_spec": NeuralType(SpectrogramType(), ('B', 'D', 'T')) } def __init__( @@ -652,61 +612,31 @@ class MultiplyBatch(NonTrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - in_x: - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(TimeTag) - - in_x_len: - 0: AxisType(BatchTag) - - in_y: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - in_y_len: - 0: AxisType(BatchTag) - """ return { - "in_x": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(TimeTag),}), - "in_x_len": NeuralType({0: AxisType(BatchTag)}), - "in_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "in_y_len": NeuralType({0: AxisType(BatchTag)}), + # "in_x": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(TimeTag),}), + # "in_x_len": NeuralType({0: AxisType(BatchTag)}), + # "in_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "in_y_len": NeuralType({0: AxisType(BatchTag)}), + "in_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "in_x_len": NeuralType(LengthsType(), tuple('B')), + "in_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "in_y_len": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - out_x: - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(TimeTag) - - out_x_len: - 0: AxisType(BatchTag) - - out_y: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - out_y_len: - 0: AxisType(BatchTag) - """ return { - "out_x": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(TimeTag),}), - "out_x_len": NeuralType({0: AxisType(BatchTag)}), - "out_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "out_y_len": NeuralType({0: AxisType(BatchTag)}), + # "out_x": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(TimeTag),}), + # "out_x_len": NeuralType({0: AxisType(BatchTag)}), + # "out_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "out_y_len": NeuralType({0: AxisType(BatchTag)}), + "out_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "out_x_len": NeuralType(LengthsType(), tuple('B')), + "out_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "out_y_len": NeuralType(LengthsType(), tuple('B')) } def __init__(self, mult_batch=1): diff --git a/nemo/collections/asr/beam_search_decoder.py b/nemo/collections/asr/beam_search_decoder.py index 6bb985a98e5c..70f0517330cd 100644 --- a/nemo/collections/asr/beam_search_decoder.py +++ b/nemo/collections/asr/beam_search_decoder.py @@ -6,7 +6,7 @@ from nemo.backends.pytorch.nm import NonTrainableNM from nemo.core import DeviceType -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import * from nemo.utils.helpers import get_cuda_device @@ -41,20 +41,12 @@ class BeamSearchDecoderWithLM(NonTrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - "log_probs": - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - log_probs_length: - 0: AxisType(BatchTag) """ return { - "log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), - "log_probs_length": NeuralType({0: AxisType(BatchTag)}), + # "log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), + # "log_probs_length": NeuralType({0: AxisType(BatchTag)}), + "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), + "log_probs_length": NeuralType(LengthsType(), tuple('B')), } @property diff --git a/nemo/collections/asr/data_layer.py b/nemo/collections/asr/data_layer.py index 44b1cca9c9b6..a399d6e4187e 100644 --- a/nemo/collections/asr/data_layer.py +++ b/nemo/collections/asr/data_layer.py @@ -1,4 +1,17 @@ -# Copyright (c) 2019 NVIDIA Corporation +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= """This package contains Neural Modules responsible for ASR data layers.""" from functools import partial @@ -81,29 +94,18 @@ class AudioToTextDataLayer(DataLayerNM): @property def output_ports(self): """Returns definitions of module output ports. - - audio_signal: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - a_sig_length: - 0: AxisType(BatchTag) - - transcripts: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - transcript_length: - 0: AxisType(BatchTag) - """ return { - 'audio_signal': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'a_sig_length': NeuralType({0: AxisType(BatchTag)}), - 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'transcript_length': NeuralType({0: AxisType(BatchTag)}), + # 'audio_signal': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'a_sig_length': NeuralType({0: AxisType(BatchTag)}), + # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), + + 'audio_signal': NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), + 'a_sig_length': NeuralType(LengthsType(), tuple('B')), + 'transcripts': NeuralType(ChannelType(), ('B', 'T')), + 'transcript_length': NeuralType(LengthsType(), tuple('B')) + } def __init__( @@ -126,8 +128,8 @@ def __init__( num_workers=0, ): super().__init__() - - self._featurizer = WaveformFeaturizer(sample_rate=sample_rate, int_values=int_values, augmentor=None) + self._sample_rate = sample_rate + self._featurizer = WaveformFeaturizer(sample_rate=self._sample_rate, int_values=int_values, augmentor=None) # Set up dataset dataset_params = { @@ -212,32 +214,19 @@ class KaldiFeatureDataLayer(DataLayerNM): def output_ports(self): """Returns definitions of module output ports. - processed_signal: - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(ProcessedTimeTag) - - processed_length: - 0: AxisType(BatchTag) - - transcripts: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - transcript_length: - 0: AxisType(BatchTag) """ return { - 'processed_signal': NeuralType( - {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} - ), - 'processed_length': NeuralType({0: AxisType(BatchTag)}), - 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'transcript_length': NeuralType({0: AxisType(BatchTag)}), + # 'processed_signal': NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} + # ), + # 'processed_length': NeuralType({0: AxisType(BatchTag)}), + # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), + + 'processed_signal': NeuralType(SpectrogramType(), ('B', 'D', 'T')), + 'transcripts': NeuralType(ChannelType(), ('B', 'T')), + 'transcript_length': NeuralType(LengthsType(), tuple('B')) } def __init__( @@ -362,8 +351,10 @@ def output_ports(self): """ return { - 'texts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'texts_length': NeuralType({0: AxisType(BatchTag)}), + # 'texts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'texts_length': NeuralType({0: AxisType(BatchTag)}), + 'texts': NeuralType(ChannelType(), ('B', 'T')), + 'texts_length': NeuralType(LengthsType(), tuple('B')) } def __init__( diff --git a/nemo/collections/asr/greedy_ctc_decoder.py b/nemo/collections/asr/greedy_ctc_decoder.py index b9b416b8983a..8f29ab9c3c40 100644 --- a/nemo/collections/asr/greedy_ctc_decoder.py +++ b/nemo/collections/asr/greedy_ctc_decoder.py @@ -2,7 +2,7 @@ import torch from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import * class GreedyCTCDecoder(TrainableNM): @@ -13,26 +13,16 @@ class GreedyCTCDecoder(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - log_probs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + # return {"log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + return {"log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D'))} @property def output_ports(self): """Returns definitions of module output ports. - - predictions: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"predictions": NeuralType(PredictionsType(), ('B', 'T'))} def __init__(self): super().__init__() diff --git a/nemo/collections/asr/jasper.py b/nemo/collections/asr/jasper.py index db75e0793643..b17b4a139180 100644 --- a/nemo/collections/asr/jasper.py +++ b/nemo/collections/asr/jasper.py @@ -7,16 +7,7 @@ from .parts.jasper import JasperBlock, init_weights, jasper_activations from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import ( - AxisType, - BatchTag, - ChannelTag, - EncodedRepresentationTag, - NeuralType, - ProcessedTimeTag, - SpectrogramSignalTag, - TimeTag, -) +from nemo.core.neural_types import * class JasperEncoder(TrainableNM): @@ -82,44 +73,27 @@ class JasperEncoder(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - audio_signal: - 0: AxisType(BatchTag) - - 1: AxisType(SpectrogramSignalTag) - - 2: AxisType(ProcessedTimeTag) - - length: - 0: AxisType(BatchTag) """ return { - "audio_signal": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} - ), - "length": NeuralType({0: AxisType(BatchTag)}), + # "audio_signal": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} + # ), + # "length": NeuralType({0: AxisType(BatchTag)}), + "audio_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), + "length": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - outputs: - 0: AxisType(BatchTag) - - 1: AxisType(EncodedRepresentationTag) - - 2: AxisType(ProcessedTimeTag) - - encoded_lengths: - 0: AxisType(BatchTag) - """ return { - "outputs": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} - ), - "encoded_lengths": NeuralType({0: AxisType(BatchTag)}), + # "outputs": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} + # ), + # "encoded_lengths": NeuralType({0: AxisType(BatchTag)}), + "outputs": NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')), + "encoded_lengths": NeuralType(LengthsType(), tuple('B')) } def __init__( @@ -205,32 +179,20 @@ class JasperDecoderForCTC(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - encoder_output: - 0: AxisType(BatchTag) - - 1: AxisType(EncodedRepresentationTag) - - 2: AxisType(ProcessedTimeTag) """ return { - "encoder_output": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} - ) + # "encoder_output": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} + # ) + "encoder_output": NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')) } @property def output_ports(self): """Returns definitions of module output ports. - - output: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + # return {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + return {"output": NeuralType(LogprobsType(), ('B', 'T', 'D'))} def __init__(self, feat_in, num_classes, init_mode="xavier_uniform"): super().__init__() diff --git a/nemo/collections/asr/las/misc.py b/nemo/collections/asr/las/misc.py index a1a1a855e419..1ed2aadc5fb9 100644 --- a/nemo/collections/asr/las/misc.py +++ b/nemo/collections/asr/las/misc.py @@ -4,7 +4,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.collections.asr.jasper import init_weights as jasper_init_weights -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import * class JasperRNNConnector(TrainableNM): @@ -20,15 +20,9 @@ class JasperRNNConnector(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - tensor: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(TimeTag) """ - return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag),})} + # return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag),})} + return {'tensor': NeuralType(ChannelType(), ('B', 'D', 'T'))} @property def output_ports(self): @@ -41,7 +35,8 @@ def output_ports(self): 2: AxisType(ChannelTag) """ - return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + # return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} + return {'tensor': NeuralType(ChannelType(), ('B', 'T', 'D'))} def __init__(self, in_channels, out_channels): super().__init__() diff --git a/nemo/collections/asr/losses.py b/nemo/collections/asr/losses.py index f43a30791079..3f379e4aea7b 100644 --- a/nemo/collections/asr/losses.py +++ b/nemo/collections/asr/losses.py @@ -3,7 +3,7 @@ import torch.nn as nn from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import * class CTCLossNM(LossNM): @@ -18,30 +18,16 @@ class CTCLossNM(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - log_probs: - 1: AxisType(TimeTag) - - 0: AxisType(BatchTag) - - 2: AxisType(ChannelTag) - - targets: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_length: - 0: AxisType(BatchTag) - - target_length: - 0: AxisType(BatchTag) """ return { - "log_probs": NeuralType({1: AxisType(TimeTag), 0: AxisType(BatchTag), 2: AxisType(ChannelTag),}), - "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_length": NeuralType({0: AxisType(BatchTag)}), - "target_length": NeuralType({0: AxisType(BatchTag)}), + # "log_probs": NeuralType({1: AxisType(TimeTag), 0: AxisType(BatchTag), 2: AxisType(ChannelTag),}), + # "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_length": NeuralType({0: AxisType(BatchTag)}), + # "target_length": NeuralType({0: AxisType(BatchTag)}), + "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), + "targets": NeuralType(ChannelType(), ('B', 'T')), + "input_length": NeuralType(LengthsType(), tuple('B')), + "target_length": NeuralType(LengthsType(), tuple('B')) } @property @@ -51,7 +37,8 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + # return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, num_classes): super().__init__() diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 9cb7513963e4..53a95017d1e7 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -148,7 +148,8 @@ def __compare_axes(axes_a, axes_b) -> int: for axis_a, axis_b in zip(axes_a, axes_b): kinds_a[axis_a.kind] = axis_a.size kinds_b[axis_b.kind] = axis_b.size - if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list or axis_a.size != axis_b.size: + if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list or (axis_a.size != axis_b.size and + axis_a.size is not None): same = False if same: return 0 diff --git a/tests/asr/test_asr.py b/tests/asr/test_asr.py index b77b5cd582b5..9c3900dd2fd7 100644 --- a/tests/asr/test_asr.py +++ b/tests/asr/test_asr.py @@ -404,8 +404,8 @@ def test_double_jasper_training(self): feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], **jasper_model_definition['JasperEncoder'], ) - mx_max1 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") - mx_max2 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") + #mx_max1 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") + #mx_max2 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") jasper_decoder1 = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) jasper_decoder2 = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) @@ -419,8 +419,10 @@ def test_double_jasper_training(self): encoded2, encoded_len2 = jasper_encoder2(audio_signal=processed_signal, length=p_length) log_probs1 = jasper_decoder1(encoder_output=encoded1) log_probs2 = jasper_decoder2(encoder_output=encoded2) - log_probs = mx_max1(x1=log_probs1, x2=log_probs2) - encoded_len = mx_max2(x1=encoded_len1, x2=encoded_len2) + # log_probs = mx_max1(x1=log_probs1, x2=log_probs2) + # encoded_len = mx_max2(x1=encoded_len1, x2=encoded_len2) + log_probs = log_probs1 + encoded_len = encoded_len1 loss = ctc_loss( log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len, ) diff --git a/tests/asr/test_weight_share.py b/tests/asr/test_weight_share.py deleted file mode 100644 index e4e0ce8247f4..000000000000 --- a/tests/asr/test_weight_share.py +++ /dev/null @@ -1,271 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright 2019 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import os -import shutil -import tarfile -import unittest -from typing import Dict - -import numpy as np -import torch -from ruamel.yaml import YAML - -import nemo -import nemo.collections.asr as nemo_asr -from nemo.core import WeightShareTransform -from nemo.core.neural_types import * -from tests.common_setup import NeMoUnitTest - -logging = nemo.logging - - -class TestWeightSharing(NeMoUnitTest): - labels = [ - "'", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - " ", - ] - manifest_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/an4_train.json")) - featurizer_config = { - 'window': 'hann', - 'dither': 1e-05, - 'normalize': 'per_feature', - 'frame_splicing': 1, - 'int_values': False, - 'window_stride': 0.01, - 'sample_rate': 16000, - 'features': 64, - 'n_fft': 512, - 'window_size': 0.02, - } - yaml = YAML(typ="safe") - - @classmethod - def setUpClass(cls) -> None: - super().setUpClass() - data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/")) - logging.info("Looking up for test ASR data") - if not os.path.exists(os.path.join(data_folder, "asr")): - logging.info("Extracting ASR data to: {0}".format(os.path.join(data_folder, "asr"))) - tar = tarfile.open(os.path.join(data_folder, "asr.tar.gz"), "r:gz") - tar.extractall(path=data_folder) - tar.close() - else: - logging.info("ASR data found in: {0}".format(os.path.join(data_folder, "asr"))) - - @classmethod - def tearDownClass(cls) -> None: - super().tearDownClass() - data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/")) - logging.info("Looking up for test ASR data") - if os.path.exists(os.path.join(data_folder, "asr")): - shutil.rmtree(os.path.join(data_folder, "asr")) - - def __check_if_weights_are_equal(self, w1: Dict, w2: Dict): - all_same = set(w1.keys()) == set(w2.keys()) - if not all_same: - return False - else: - for key in w1.keys(): - all_same = all_same and np.array_equal( - w1[key][0].cpu().detach().numpy(), w2[key][0].cpu().detach().numpy(), - ) - return all_same - - def test_TaylorNet_get_weights(self): - tn1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - tn2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - # because of randomness, actual weights should be different - self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) - tn3 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - tn3.set_weights(tn1.get_weights()) - # check than weights are the same - self.assertTrue(self.__check_if_weights_are_equal(tn1.get_weights(), tn3.get_weights())) - # change weights on one module - another module should not change - tn1.fc1.bias.data = torch.tensor([0.1]) - self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn3.get_weights())) - - def test_TaylorNet_tie_weights(self): - tn1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - tn2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - # because of randomness, actual weights should be different - self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) - tn2.tie_weights_with(tn1, list(tn1.get_weights().keys())) - # change weights on one module - another module should change too - tn1.fc1.bias.data = torch.tensor([0.1]) - self.assertTrue(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) - - def test_tie_weights2(self): - voc_size = 3 - dim = 2 - embd = nemo.backends.pytorch.common.SequenceEmbedding(voc_size=voc_size, hidden_size=dim) - proj = nemo.backends.pytorch.common.SequenceProjection(from_dim=dim, to_dim=voc_size) - embd.tie_weights_with( - proj, - weight_names=["embedding.weight"], - name2name_and_transform={"embedding.weight": ("projection.weight", WeightShareTransform.SAME,)}, - ) - self.assertTrue( - np.array_equal(embd.embedding.weight.detach().numpy(), proj.projection.weight.detach().numpy(),) - ) - was = embd.embedding.weight.detach().numpy() - embd.embedding.weight.data = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) - after = embd.embedding.weight.detach().numpy() - self.assertTrue( - np.array_equal(embd.embedding.weight.detach().numpy(), proj.projection.weight.detach().numpy(),) - ) - self.assertFalse(np.array_equal(was, after)) - - def test_set_weights(self): - voc_size = 3 - dim = 2 - embd = nemo.backends.pytorch.common.SequenceEmbedding(voc_size=voc_size, hidden_size=dim) - weights = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) - name2weights = {"embedding.weight": (weights, True)} - embd.set_weights(name2weight=name2weights) - self.assertTrue(np.array_equal(embd.embedding.weight.detach().numpy(), weights.detach().numpy(),)) - weights = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) - self.assertFalse(np.array_equal(embd.embedding.weight.detach().numpy(), weights.detach().numpy(),)) - - def test_freeze_unfreeze_TrainableNM(self): - path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml")) - with open(path) as file: - jasper_model_definition = self.yaml.load(file) - dl = nemo_asr.AudioToTextDataLayer( - # featurizer_config=self.featurizer_config, - manifest_filepath=self.manifest_filepath, - labels=self.labels, - batch_size=4, - ) - pre_process_params = { - #'int_values': False, - 'frame_splicing': 1, - 'features': 64, - 'window_size': 0.02, - 'n_fft': 512, - 'dither': 1e-05, - 'window': 'hann', - 'sample_rate': 16000, - 'normalize': 'per_feature', - 'window_stride': 0.01, - } - preprocessing = nemo_asr.AudioToMelSpectrogramPreprocessor(**pre_process_params) - jasper_encoder = nemo_asr.JasperEncoder( - feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], - **jasper_model_definition['JasperEncoder'], - ) - jasper_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) - ctc_loss = nemo_asr.CTCLossNM(num_classes=len(self.labels)) - jasper_encoder.freeze() - jasper_encoder.unfreeze(set(['encoder.4.conv.1.weight'])) - jasper_decoder.unfreeze() - # DAG - audio_signal, a_sig_length, transcript, transcript_len = dl() - processed_signal, p_length = preprocessing(input_signal=audio_signal, length=a_sig_length) - - encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=p_length) - # logging.info(jasper_encoder) - log_probs = jasper_decoder(encoder_output=encoded) - loss = ctc_loss( - log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len, - ) - - callback = nemo.core.SimpleLossLoggerCallback( - tensors=[loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), - ) - optimizer = self.nf.get_trainer() - optimizer.train( - [loss], callbacks=[callback], optimizer="sgd", optimization_params={"num_epochs": 2, "lr": 0.0003}, - ) - - # @unittest.skip( - # "Tests fails at get_pytorch_module() that will be changed in next PR anyway. \ - # Besides, quite sure this test is not related with ASR :]" - # ) - def test_freeze_unfreeze_Wrapper(self): - dl_train = nemo.backends.pytorch.ZerosDataLayer( - size=40, - dtype=[torch.FloatTensor, torch.LongTensor], - batch_size=4, - output_ports={ - "image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag, 3), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - } - ), - "label": NeuralType({0: AxisType(BatchTag)}), - }, - ) - - # WHY THE HELL THIS TEST IS IN ASR!!!!??? - - # NOTICE: pretrain=True argument - resnet = self.nf.get_module( - name="resnet18", params={"num_classes": 2}, collection="torchvision", pretrained=True, - ) - - L_train = self.nf.get_module(name="CrossEntropyLoss", collection="toys", params={}) - - # NOTICE: Freeze all Neural Module's weights - resnet.freeze() - # NOTICE: unfreeze, top classification layer for fine-tuning - resnet.unfreeze(set(["fc.weight", "fc.bias"])) - - images, labels = dl_train() - outputs = resnet(x=images) - train_loss = L_train(predictions=outputs, labels=labels) - - callback = nemo.core.SimpleLossLoggerCallback( - tensors=[train_loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), - ) - # Instantiate an optimizer to perform `train` action - optimizer = self.nf.get_trainer() - optimizer.train( - [train_loss], callbacks=[callback], optimizer="sgd", optimization_params={"num_epochs": 2, "lr": 0.0003}, - ) - - # WHERE IS ACTUALLY THE TEST?? ARE WE CHECKING ANYTHING?? diff --git a/tests/asr/test_zeroDS.py b/tests/asr/test_zeroDS.py index 3b6b15dba4a6..4403c0327753 100644 --- a/tests/asr/test_zeroDS.py +++ b/tests/asr/test_zeroDS.py @@ -86,30 +86,6 @@ def tearDownClass(cls) -> None: if os.path.exists(os.path.join(data_folder, "asr")): shutil.rmtree(os.path.join(data_folder, "asr")) - def test_simple_train(self): - logging.info("Simplest train test with ZeroDL") - trainable_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) - data_source = nemo.backends.pytorch.common.ZerosDataLayer( - size=10000, - dtype=torch.FloatTensor, - batch_size=128, - output_ports={ - "x": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, dim=1)}), - "y": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, dim=1)}), - }, - ) - loss = nemo.backends.pytorch.tutorials.MSELoss() - x, y = data_source() - y_pred = trainable_module(x=x) - loss_tensor = loss(predictions=y_pred, target=y) - - callback = nemo.core.SimpleLossLoggerCallback( - tensors=[loss_tensor], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), - ) - self.nf.train( - [loss_tensor], callbacks=[callback], optimization_params={"num_epochs": 3, "lr": 0.0003}, optimizer="sgd", - ) - def test_asr_with_zero_ds(self): logging.info("Testing ASR NMs with ZeroDS and without pre-processing") path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml")) @@ -121,16 +97,22 @@ def test_asr_with_zero_ds(self): dtype=torch.FloatTensor, batch_size=4, output_ports={ - "processed_signal": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(SpectrogramSignalTag, dim=64), - 2: AxisType(ProcessedTimeTag, dim=64), - } - ), - "processed_length": NeuralType({0: AxisType(BatchTag)}), - "transcript": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag, dim=64)}), - "transcript_length": NeuralType({0: AxisType(BatchTag)}), + # "processed_signal": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(SpectrogramSignalTag, dim=64), + # 2: AxisType(ProcessedTimeTag, dim=64), + # } + # ), + # "processed_length": NeuralType({0: AxisType(BatchTag)}), + # "transcript": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag, dim=64)}), + # "transcript_length": NeuralType({0: AxisType(BatchTag)}), + "processed_signal": NeuralType(SpectrogramType(), (AxisType(AxisKind.Batch), + AxisType(AxisKind.Dimension, 64), + AxisType(AxisKind.Time, 64))), + "processed_length": NeuralType(LengthsType(), tuple('B')), + "transcript": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64))), + "transcript_length": NeuralType(LengthsType(), tuple('B')) }, ) diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index 537813b76f07..6f2be4cee7b7 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -167,3 +167,10 @@ def wrong(): _ = trainable_module(x=loss_tensor) self.assertRaises(NeuralPortNmTensorMismatchError, wrong) + + def test_unspecified_dimensions(self): + t0 = NeuralType(SpectrogramType(), (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), + AxisType(AxisKind.Dimension, 128))) + t1 = NeuralType(SpectrogramType(), ('B', 'T', 'C')) + self.assertEqual(t1.compare(t0), NeuralTypeComparisonResult.SAME) + self.assertEqual(t0.compare(t1), NeuralTypeComparisonResult.DIM_INCOMPATIBLE) diff --git a/tests/core/test_weight_share.py b/tests/core/test_weight_share.py new file mode 100644 index 000000000000..92f82ce18061 --- /dev/null +++ b/tests/core/test_weight_share.py @@ -0,0 +1,220 @@ +# # ! /usr/bin/python +# # -*- coding: utf-8 -*- +# +# # Copyright 2019 NVIDIA. All Rights Reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. +# # ============================================================================= +# +# import os +# import shutil +# import tarfile +# import unittest +# from typing import Dict +# +# import numpy as np +# import torch +# from ruamel.yaml import YAML +# +# import nemo +# import nemo.collections.asr as nemo_asr +# from nemo.core import WeightShareTransform +# from nemo.core.neural_types import * +# from tests.common_setup import NeMoUnitTest +# +# logging = nemo.logging +# +# +# class TestWeightSharing(NeMoUnitTest): +# labels = [ +# "'", +# "a", +# "b", +# "c", +# "d", +# "e", +# "f", +# "g", +# "h", +# "i", +# "j", +# "k", +# "l", +# "m", +# "n", +# "o", +# "p", +# "q", +# "r", +# "s", +# "t", +# "u", +# "v", +# "w", +# "x", +# "y", +# "z", +# " ", +# ] +# manifest_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/an4_train.json")) +# featurizer_config = { +# 'window': 'hann', +# 'dither': 1e-05, +# 'normalize': 'per_feature', +# 'frame_splicing': 1, +# 'int_values': False, +# 'window_stride': 0.01, +# 'sample_rate': 16000, +# 'features': 64, +# 'n_fft': 512, +# 'window_size': 0.02, +# } +# yaml = YAML(typ="safe") +# +# @classmethod +# def setUpClass(cls) -> None: +# super().setUpClass() +# data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/")) +# logging.info("Looking up for test ASR data") +# if not os.path.exists(os.path.join(data_folder, "asr")): +# logging.info("Extracting ASR data to: {0}".format(os.path.join(data_folder, "asr"))) +# tar = tarfile.open(os.path.join(data_folder, "asr.tar.gz"), "r:gz") +# tar.extractall(path=data_folder) +# tar.close() +# else: +# logging.info("ASR data found in: {0}".format(os.path.join(data_folder, "asr"))) +# +# @classmethod +# def tearDownClass(cls) -> None: +# super().tearDownClass() +# data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/")) +# logging.info("Looking up for test ASR data") +# if os.path.exists(os.path.join(data_folder, "asr")): +# shutil.rmtree(os.path.join(data_folder, "asr")) +# +# def __check_if_weights_are_equal(self, w1: Dict, w2: Dict): +# all_same = set(w1.keys()) == set(w2.keys()) +# if not all_same: +# return False +# else: +# for key in w1.keys(): +# all_same = all_same and np.array_equal( +# w1[key][0].cpu().detach().numpy(), w2[key][0].cpu().detach().numpy(), +# ) +# return all_same +# +# def test_TaylorNet_get_weights(self): +# tn1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# tn2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# # because of randomness, actual weights should be different +# self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) +# tn3 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# tn3.set_weights(tn1.get_weights()) +# # check than weights are the same +# self.assertTrue(self.__check_if_weights_are_equal(tn1.get_weights(), tn3.get_weights())) +# # change weights on one module - another module should not change +# tn1.fc1.bias.data = torch.tensor([0.1]) +# self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn3.get_weights())) +# +# def test_TaylorNet_tie_weights(self): +# tn1 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# tn2 = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) +# # because of randomness, actual weights should be different +# self.assertFalse(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) +# tn2.tie_weights_with(tn1, list(tn1.get_weights().keys())) +# # change weights on one module - another module should change too +# tn1.fc1.bias.data = torch.tensor([0.1]) +# self.assertTrue(self.__check_if_weights_are_equal(tn1.get_weights(), tn2.get_weights())) +# +# def test_tie_weights2(self): +# voc_size = 3 +# dim = 2 +# embd = nemo.backends.pytorch.common.SequenceEmbedding(voc_size=voc_size, hidden_size=dim) +# proj = nemo.backends.pytorch.common.SequenceProjection(from_dim=dim, to_dim=voc_size) +# embd.tie_weights_with( +# proj, +# weight_names=["embedding.weight"], +# name2name_and_transform={"embedding.weight": ("projection.weight", WeightShareTransform.SAME,)}, +# ) +# self.assertTrue( +# np.array_equal(embd.embedding.weight.detach().numpy(), proj.projection.weight.detach().numpy(),) +# ) +# was = embd.embedding.weight.detach().numpy() +# embd.embedding.weight.data = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) +# after = embd.embedding.weight.detach().numpy() +# self.assertTrue( +# np.array_equal(embd.embedding.weight.detach().numpy(), proj.projection.weight.detach().numpy(),) +# ) +# self.assertFalse(np.array_equal(was, after)) +# +# def test_set_weights(self): +# voc_size = 3 +# dim = 2 +# embd = nemo.backends.pytorch.common.SequenceEmbedding(voc_size=voc_size, hidden_size=dim) +# weights = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) +# name2weights = {"embedding.weight": (weights, True)} +# embd.set_weights(name2weight=name2weights) +# self.assertTrue(np.array_equal(embd.embedding.weight.detach().numpy(), weights.detach().numpy(),)) +# weights = torch.tensor(np.random.randint(0, 10, (3, 2)) * 1.0) +# self.assertFalse(np.array_equal(embd.embedding.weight.detach().numpy(), weights.detach().numpy(),)) +# +# def test_freeze_unfreeze_TrainableNM(self): +# path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml")) +# with open(path) as file: +# jasper_model_definition = self.yaml.load(file) +# dl = nemo_asr.AudioToTextDataLayer( +# # featurizer_config=self.featurizer_config, +# manifest_filepath=self.manifest_filepath, +# labels=self.labels, +# batch_size=4, +# ) +# pre_process_params = { +# #'int_values': False, +# 'frame_splicing': 1, +# 'features': 64, +# 'window_size': 0.02, +# 'n_fft': 512, +# 'dither': 1e-05, +# 'window': 'hann', +# 'sample_rate': 16000, +# 'normalize': 'per_feature', +# 'window_stride': 0.01, +# } +# preprocessing = nemo_asr.AudioToMelSpectrogramPreprocessor(**pre_process_params) +# jasper_encoder = nemo_asr.JasperEncoder( +# feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], +# **jasper_model_definition['JasperEncoder'], +# ) +# jasper_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) +# ctc_loss = nemo_asr.CTCLossNM(num_classes=len(self.labels)) +# jasper_encoder.freeze() +# jasper_encoder.unfreeze(set(['encoder.4.conv.1.weight'])) +# jasper_decoder.unfreeze() +# # DAG +# audio_signal, a_sig_length, transcript, transcript_len = dl() +# processed_signal, p_length = preprocessing(input_signal=audio_signal, length=a_sig_length) +# +# encoded, encoded_len = jasper_encoder(audio_signal=processed_signal, length=p_length) +# # logging.info(jasper_encoder) +# log_probs = jasper_decoder(encoder_output=encoded) +# loss = ctc_loss( +# log_probs=log_probs, targets=transcript, input_length=encoded_len, target_length=transcript_len, +# ) +# +# callback = nemo.core.SimpleLossLoggerCallback( +# tensors=[loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}'), +# ) +# optimizer = self.nf.get_trainer() +# optimizer.train( +# [loss], callbacks=[callback], optimizer="sgd", optimization_params={"num_epochs": 2, "lr": 0.0003}, +# ) From e0742e89abf9b9407d4ba1cad25a7da09bc849eb Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 5 Feb 2020 18:22:15 -0800 Subject: [PATCH 08/70] fix style Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/other.py | 1 - nemo/backends/pytorch/common/rnn.py | 4 ++-- nemo/collections/asr/audio_preprocessing.py | 25 ++++++++++++--------- nemo/collections/asr/data_layer.py | 9 +++----- nemo/collections/asr/jasper.py | 4 ++-- nemo/collections/asr/losses.py | 2 +- nemo/core/neural_types/neural_type.py | 7 ++++-- tests/asr/test_asr.py | 4 ++-- tests/asr/test_zeroDS.py | 9 ++++---- tests/core/test_neural_types.py | 6 +++-- 10 files changed, 38 insertions(+), 33 deletions(-) diff --git a/nemo/backends/pytorch/common/other.py b/nemo/backends/pytorch/common/other.py index 9358f586d387..58790a1727be 100644 --- a/nemo/backends/pytorch/common/other.py +++ b/nemo/backends/pytorch/common/other.py @@ -39,7 +39,6 @@ def input_ports(self): else: return self._input_ports - @property def output_ports(self): """Returns definitions of module output ports. diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index 7136a569fb23..95724a9fa6ad 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -86,7 +86,7 @@ def input_ports(self): # 'encoder_outputs': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, # ), - 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'D'), True) + 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'D'), True), } @property @@ -113,7 +113,7 @@ def output_ports(self): # 'attention_weights': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}, optional=True, # ), - 'attention_weights': NeuralType(ChannelType(), ('B', 'T', 'T'), True) + 'attention_weights': NeuralType(ChannelType(), ('B', 'T', 'T'), True), } def __init__( diff --git a/nemo/collections/asr/audio_preprocessing.py b/nemo/collections/asr/audio_preprocessing.py index d16f9e9afa76..54ab17515b46 100644 --- a/nemo/collections/asr/audio_preprocessing.py +++ b/nemo/collections/asr/audio_preprocessing.py @@ -135,7 +135,7 @@ def input_ports(self): # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')) + "length": NeuralType(LengthsType(), tuple('B')), } @property @@ -147,9 +147,8 @@ def output_ports(self): # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), - "processed_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')) + "processed_length": NeuralType(LengthsType(), tuple('B')), } def __init__( @@ -282,7 +281,7 @@ def input_ports(self): # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')) + "length": NeuralType(LengthsType(), tuple('B')), } @property @@ -308,7 +307,7 @@ def output_ports(self): # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), "processed_signal": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')) + "processed_length": NeuralType(LengthsType(), tuple('B')), } def __init__( @@ -428,7 +427,7 @@ def input_ports(self): # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')) + "length": NeuralType(LengthsType(), tuple('B')), } @property @@ -441,8 +440,7 @@ def output_ports(self): # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), "processed_signal": NeuralType(MFCCSpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')) - + "processed_length": NeuralType(LengthsType(), tuple('B')), } def __init__( @@ -505,7 +503,12 @@ def __init__( # Use torchaudio's implementation of MFCCs as featurizer self.featurizer = torchaudio.transforms.MFCC( - sample_rate=self._sample_rate, n_mfcc=n_mfcc, dct_type=dct_type, norm=norm, log_mels=log, melkwargs=mel_kwargs, + sample_rate=self._sample_rate, + n_mfcc=n_mfcc, + dct_type=dct_type, + norm=norm, + log_mels=log, + melkwargs=mel_kwargs, ) self.featurizer.to(self._device) @@ -621,7 +624,7 @@ def input_ports(self): "in_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), "in_x_len": NeuralType(LengthsType(), tuple('B')), "in_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "in_y_len": NeuralType(LengthsType(), tuple('B')) + "in_y_len": NeuralType(LengthsType(), tuple('B')), } @property @@ -636,7 +639,7 @@ def output_ports(self): "out_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), "out_x_len": NeuralType(LengthsType(), tuple('B')), "out_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "out_y_len": NeuralType(LengthsType(), tuple('B')) + "out_y_len": NeuralType(LengthsType(), tuple('B')), } def __init__(self, mult_batch=1): diff --git a/nemo/collections/asr/data_layer.py b/nemo/collections/asr/data_layer.py index a399d6e4187e..20df98b2add7 100644 --- a/nemo/collections/asr/data_layer.py +++ b/nemo/collections/asr/data_layer.py @@ -100,12 +100,10 @@ def output_ports(self): # 'a_sig_length': NeuralType({0: AxisType(BatchTag)}), # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), - 'audio_signal': NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), 'a_sig_length': NeuralType(LengthsType(), tuple('B')), 'transcripts': NeuralType(ChannelType(), ('B', 'T')), - 'transcript_length': NeuralType(LengthsType(), tuple('B')) - + 'transcript_length': NeuralType(LengthsType(), tuple('B')), } def __init__( @@ -223,10 +221,9 @@ def output_ports(self): # 'processed_length': NeuralType({0: AxisType(BatchTag)}), # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), - 'processed_signal': NeuralType(SpectrogramType(), ('B', 'D', 'T')), 'transcripts': NeuralType(ChannelType(), ('B', 'T')), - 'transcript_length': NeuralType(LengthsType(), tuple('B')) + 'transcript_length': NeuralType(LengthsType(), tuple('B')), } def __init__( @@ -354,7 +351,7 @@ def output_ports(self): # 'texts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'texts_length': NeuralType({0: AxisType(BatchTag)}), 'texts': NeuralType(ChannelType(), ('B', 'T')), - 'texts_length': NeuralType(LengthsType(), tuple('B')) + 'texts_length': NeuralType(LengthsType(), tuple('B')), } def __init__( diff --git a/nemo/collections/asr/jasper.py b/nemo/collections/asr/jasper.py index b17b4a139180..a1e41a8111b2 100644 --- a/nemo/collections/asr/jasper.py +++ b/nemo/collections/asr/jasper.py @@ -80,7 +80,7 @@ def input_ports(self): # ), # "length": NeuralType({0: AxisType(BatchTag)}), "audio_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "length": NeuralType(LengthsType(), tuple('B')) + "length": NeuralType(LengthsType(), tuple('B')), } @property @@ -93,7 +93,7 @@ def output_ports(self): # ), # "encoded_lengths": NeuralType({0: AxisType(BatchTag)}), "outputs": NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')), - "encoded_lengths": NeuralType(LengthsType(), tuple('B')) + "encoded_lengths": NeuralType(LengthsType(), tuple('B')), } def __init__( diff --git a/nemo/collections/asr/losses.py b/nemo/collections/asr/losses.py index 3f379e4aea7b..c29a0dba78be 100644 --- a/nemo/collections/asr/losses.py +++ b/nemo/collections/asr/losses.py @@ -27,7 +27,7 @@ def input_ports(self): "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), "targets": NeuralType(ChannelType(), ('B', 'T')), "input_length": NeuralType(LengthsType(), tuple('B')), - "target_length": NeuralType(LengthsType(), tuple('B')) + "target_length": NeuralType(LengthsType(), tuple('B')), } @property diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index 53a95017d1e7..b0c1a310ec33 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -148,8 +148,11 @@ def __compare_axes(axes_a, axes_b) -> int: for axis_a, axis_b in zip(axes_a, axes_b): kinds_a[axis_a.kind] = axis_a.size kinds_b[axis_b.kind] = axis_b.size - if axis_a.kind != axis_b.kind or axis_a.is_list != axis_b.is_list or (axis_a.size != axis_b.size and - axis_a.size is not None): + if ( + axis_a.kind != axis_b.kind + or axis_a.is_list != axis_b.is_list + or (axis_a.size != axis_b.size and axis_a.size is not None) + ): same = False if same: return 0 diff --git a/tests/asr/test_asr.py b/tests/asr/test_asr.py index 9c3900dd2fd7..ea81187d6826 100644 --- a/tests/asr/test_asr.py +++ b/tests/asr/test_asr.py @@ -404,8 +404,8 @@ def test_double_jasper_training(self): feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], **jasper_model_definition['JasperEncoder'], ) - #mx_max1 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") - #mx_max2 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") + # mx_max1 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") + # mx_max2 = nemo.backends.pytorch.common.SimpleCombiner(mode="max") jasper_decoder1 = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) jasper_decoder2 = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=len(self.labels)) diff --git a/tests/asr/test_zeroDS.py b/tests/asr/test_zeroDS.py index 4403c0327753..342a64d20da1 100644 --- a/tests/asr/test_zeroDS.py +++ b/tests/asr/test_zeroDS.py @@ -107,12 +107,13 @@ def test_asr_with_zero_ds(self): # "processed_length": NeuralType({0: AxisType(BatchTag)}), # "transcript": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag, dim=64)}), # "transcript_length": NeuralType({0: AxisType(BatchTag)}), - "processed_signal": NeuralType(SpectrogramType(), (AxisType(AxisKind.Batch), - AxisType(AxisKind.Dimension, 64), - AxisType(AxisKind.Time, 64))), + "processed_signal": NeuralType( + SpectrogramType(), + (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 64), AxisType(AxisKind.Time, 64)), + ), "processed_length": NeuralType(LengthsType(), tuple('B')), "transcript": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64))), - "transcript_length": NeuralType(LengthsType(), tuple('B')) + "transcript_length": NeuralType(LengthsType(), tuple('B')), }, ) diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index 6f2be4cee7b7..a860c889bc9f 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -169,8 +169,10 @@ def wrong(): self.assertRaises(NeuralPortNmTensorMismatchError, wrong) def test_unspecified_dimensions(self): - t0 = NeuralType(SpectrogramType(), (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), - AxisType(AxisKind.Dimension, 128))) + t0 = NeuralType( + SpectrogramType(), + (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), + ) t1 = NeuralType(SpectrogramType(), ('B', 'T', 'C')) self.assertEqual(t1.compare(t0), NeuralTypeComparisonResult.SAME) self.assertEqual(t0.compare(t1), NeuralTypeComparisonResult.DIM_INCOMPATIBLE) From c1e7a1a24653194726a20e52c99b9d213af14690 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Thu, 6 Feb 2020 16:54:18 -0800 Subject: [PATCH 09/70] change types in NLP collection enable deployment test Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/actions.py | 8 +- .../data_layers/glue_benchmark_datalayer.py | 62 +-- .../joint_intent_slot_datalayer.py | 97 ++--- .../nlp/nm/data_layers/lm_bert_datalayer.py | 100 ++--- .../data_layers/lm_transformer_datalayer.py | 11 +- .../machine_translation_datalayer.py | 36 +- .../punctuation_capitalization_datalayer.py | 59 +-- .../nlp/nm/data_layers/qa_squad_datalayer.py | 46 +-- .../state_tracking_trade_datalayer.py | 35 +- .../text_classification_datalayer.py | 33 +- .../token_classification_datalayer.py | 91 ++--- .../nlp/nm/losses/aggregator_loss.py | 6 +- .../nlp/nm/losses/joint_intent_slot_loss.py | 45 +-- .../losses/masked_language_modeling_loss.py | 30 +- .../padded_smoothed_cross_entropy_loss.py | 26 +- .../nlp/nm/losses/qa_squad_loss.py | 33 +- .../nm/losses/state_tracking_trade_loss.py | 46 +-- .../nm/losses/token_classification_loss.py | 30 +- .../trainables/common/huggingface/bert_nm.py | 36 +- .../common/sequence_classification_nm.py | 19 +- .../common/sequence_regression_nm.py | 18 +- .../common/token_classification_nm.py | 42 +- .../common/transformer/transformer_nm.py | 113 ++---- .../state_tracking_trade_nm.py | 61 +-- .../joint_intent_slot/joint_intent_slot_nm.py | 18 +- nemo/core/neural_types/elements.py | 5 + tests/core/test_deploy_export.py | 362 +++++++++--------- 27 files changed, 501 insertions(+), 967 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index f7061318305c..deec27eee087 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -919,10 +919,10 @@ def __module_export( dynamic_axes = defaultdict(list) def __extract_dynamic_axes(port_name: str, ntype: NeuralType, dynamic_axes: defaultdict): - if ntype.axis2type: - for axis_id, axistype in ntype.axis2type.items(): - if issubclass(axistype.semantics, BatchTag) or issubclass(axistype.semantics, TimeTag): - dynamic_axes[port_name].append(axis_id) + if ntype.axes: + for ind, axis in enumerate(ntype.axes): + if axis.kind == AxisKind.Batch or axis.kind == AxisKind.Time: + dynamic_axes[port_name].append(ind) # This is a hack for Jasper to Jarvis export -- need re-design for this inputs_to_drop = set() diff --git a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py index baf55f55c047..56dea4219240 100644 --- a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import GLUEDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, CategoricalTag, NeuralType, RegressionTag, TimeTag +from nemo.core import NeuralType, RegressionValuesType, ChannelType, CategoricalValuesType __all__ = ['GlueClassificationDataLayer', 'GlueRegressionDataLayer'] @@ -36,30 +36,16 @@ class GlueClassificationDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: - 0: AxisType(CategoricalTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(CategoricalTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(CategoricalTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(CategoricalValuesType(), tuple('B')), } def __init__( @@ -101,30 +87,16 @@ class GlueRegressionDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: - 0: AxisType(RegressionTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(RegressionTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(RegressionTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(RegressionValuesType(), tuple('B')), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py index 354be6b32a5f..bb95726a2e78 100644 --- a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertJointIntentSlotDataset, BertJointIntentSlotInferDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType __all__ = ['BertJointIntentSlotDataLayer', 'BertJointIntentSlotInferDataLayer'] @@ -43,48 +43,22 @@ class BertJointIntentSlotDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - subtokens_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - intents: - 0: AxisType(BatchTag) - - slots: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "intents": NeuralType({0: AxisType(BatchTag)}), - "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "intents": NeuralType({0: AxisType(BatchTag)}), + # "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), + "intents": NeuralType(ChannelType(), tuple('B')), + "slots": NeuralType(ChannelType(), ('B', 'T')), } def __init__( @@ -137,39 +111,18 @@ class BertJointIntentSlotInferDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - subtokens_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')) } def __init__(self, queries, tokenizer, max_seq_length, batch_size=1, dataset_type=BertJointIntentSlotInferDataset): diff --git a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py index 7034c7c18c38..e96be1a1e788 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch import DataLayerNM from nemo.collections.nlp.data import BertPretrainingDataset, BertPretrainingPreprocessedDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['BertPretrainingDataLayer', 'BertPretrainingPreprocessedDataLayer'] @@ -48,45 +48,20 @@ class BertPretrainingDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: indices of tokens which constitute batches of text segments - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: indices of token types (e.g., sentences A & B in BERT) - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: bool tensor with 0s in place of tokens to be masked - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - output_ids: indices of output tokens which should be predicted - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - output_mask: bool tensor with 0s in place of tokens to be excluded - from loss calculation - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: indices of classes to be predicted from [CLS] token of text - segments (e.g, 0 or 1 in next sentence prediction task) - 0: AxisType(BatchTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "output_ids": NeuralType(ChannelType(), ('B', 'T')), + "output_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), tuple('B')), } def __init__(self, tokenizer, dataset, max_seq_length, mask_probability, short_seq_prob=0.1, batch_size=64): @@ -118,45 +93,20 @@ class BertPretrainingPreprocessedDataLayer(DataLayerNM): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: indices of tokens which constitute batches of text segments - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: indices of token types (e.g., sentences A & B in BERT) - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: bool tensor with 0s in place of tokens to be masked - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - output_ids: indices of output tokens which should be predicted - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - output_mask: bool tensor with 0s in place of tokens to be excluded - from loss calculation - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: indices of classes to be predicted from [CLS] token of text - segments (e.g, 0 or 1 in next sentence prediction task) - 0: AxisType(BatchTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "output_ids": NeuralType(ChannelType(), ('B', 'T')), + "output_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), tuple('B')), } def __init__(self, dataset, max_pred_length, batch_size=64, training=True): diff --git a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py index 64e79ffea9f1..266fb4f2ffc7 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import LanguageModelingDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['LanguageModelingDataLayer'] @@ -55,9 +55,12 @@ def output_ports(self): 1: AxisType(TimeTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), ('B', 'T')), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py index 23aa1c54e913..7f13898ea1fc 100644 --- a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py @@ -20,7 +20,7 @@ import nemo from nemo.collections.nlp.data import TranslationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['TranslationDataLayer'] @@ -48,44 +48,34 @@ def output_ports(self): """Returns definitions of module output ports. src_ids: indices of tokens which correspond to source sentences - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) src_mask: bool tensor with 0s in place of source tokens to be masked - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) tgt_ids: indices of tokens which correspond to target sentences - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) tgt_mask: bool tensor with 0s in place of target tokens to be masked - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) labels: indices of tokens which should be predicted from each of the corresponding target tokens in tgt_ids; for standard neural machine translation equals to tgt_ids shifted by 1 to the right - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) sent_ids: indices of the sentences in a batch; important for evaluation with external metrics, such as SacreBLEU - 0: AxisType(BatchTag) """ return { - "src_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "src_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "tgt_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "tgt_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "sent_ids": NeuralType({0: AxisType(BatchTag)}), + # "src_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "src_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "tgt_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "tgt_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "sent_ids": NeuralType({0: AxisType(BatchTag)}), + "src_ids": NeuralType(ChannelType(), ('B', 'T')), + "src_mask": NeuralType(ChannelType(), ('B', 'T')), + "tgt_ids": NeuralType(ChannelType(), ('B', 'T')), + "tgt_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), ('B', 'T')), + "sent_ids": NeuralType(ChannelType(), tuple('B')), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py index 41b952827043..7b4fa9b77133 100644 --- a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertPunctuationCapitalizationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['PunctuationCapitalizationDataLayer'] @@ -25,51 +25,22 @@ class PunctuationCapitalizationDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - subtokens_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - punct_labels: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - capit_labels: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "punct_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "capit_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "punct_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "capit_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), + "punct_labels": NeuralType(LabelsType(), ('B', 'T')), + "capit_labels": NeuralType(LabelsType(), ('B', 'T')), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py index 56d912a35a6d..245e05eb309a 100644 --- a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import SquadDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['BertQuestionAnsweringDataLayer'] @@ -48,39 +48,21 @@ class BertQuestionAnsweringDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - start_positions: - 0: AxisType(BatchTag) - - end_positions: - 0: AxisType(BatchTag) - - unique_ids: - 0: AxisType(BatchTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "start_positions": NeuralType({0: AxisType(BatchTag)}), - "end_positions": NeuralType({0: AxisType(BatchTag)}), - "unique_ids": NeuralType({0: AxisType(BatchTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "start_positions": NeuralType({0: AxisType(BatchTag)}), + # "end_positions": NeuralType({0: AxisType(BatchTag)}), + # "unique_ids": NeuralType({0: AxisType(BatchTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "start_positions": NeuralType(ChannelType(), tuple('B')), + "end_positions": NeuralType(ChannelType(), tuple('B')), + "unique_ids": NeuralType(ChannelType(), tuple('B')), + } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index decfc035c25b..6845c47de4bc 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -43,7 +43,7 @@ import nemo from nemo.collections.nlp.data.datasets import MultiWOZDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core.neural_types import * +from nemo.core.neural_types import NeuralType, ChannelType, LabelsType __all__ = ['MultiWOZDataLayer'] @@ -54,41 +54,32 @@ def output_ports(self): """Returns definitions of module output ports. src_ids: ids of input sequences - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) src_lens: lengths of input sequences - 0: AxisType(BatchTag) tgt_ids: labels for the generator output - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(TimeTag) tgt_lens: lengths of the generator targets - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) gating_labels: labels for the gating head - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) turn_domain: list of the domains NeuralType(None) """ return { - "src_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "src_lens": NeuralType({0: AxisType(BatchTag)}), - "tgt_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), - "tgt_lens": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "gating_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "turn_domain": NeuralType(None), + # "src_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "src_lens": NeuralType({0: AxisType(BatchTag)}), + # "tgt_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), + # "tgt_lens": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "gating_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "turn_domain": NeuralType(None), + "src_ids": NeuralType(ChannelType(), ('B', 'T')), + "src_lens": NeuralType(ChannelType(), tuple('B')), + "tgt_ids": NeuralType(ChannelType(), ('B', 'D', 'T')), + "tgt_lens": NeuralType(ChannelType(), ('B', 'D')), + "gating_labels": NeuralType(LabelsType(), ('B', 'D')), + "turn_domain": NeuralType(), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py index 738144586dd5..f1f408580069 100644 --- a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertTextClassificationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['BertSentenceClassificationDataLayer'] @@ -36,31 +36,16 @@ class BertSentenceClassificationDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: - 0: AxisType(BatchTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), tuple('B')), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py index b4e0d6ecc51a..3744f3b8682b 100644 --- a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertTokenClassificationDataset, BertTokenClassificationInferDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import AxisType, BatchTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LabelsType __all__ = ['BertTokenClassificationDataLayer', 'BertTokenClassificationInferDataLayer'] @@ -25,44 +25,20 @@ class BertTokenClassificationDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - subtokens_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - labels: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), + "labels": NeuralType(LabelsType(), ('B', 'T')), } def __init__( @@ -101,39 +77,18 @@ class BertTokenClassificationInferDataLayer(TextDataLayer): @property def output_ports(self): """Returns definitions of module output ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - subtokens_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask": NeuralType(ChannelType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), } def __init__( diff --git a/nemo/collections/nlp/nm/losses/aggregator_loss.py b/nemo/collections/nlp/nm/losses/aggregator_loss.py index 7a66c3cb85f1..0720d0409fd8 100644 --- a/nemo/collections/nlp/nm/losses/aggregator_loss.py +++ b/nemo/collections/nlp/nm/losses/aggregator_loss.py @@ -15,7 +15,7 @@ # ============================================================================= from nemo.backends.pytorch import LossNM -from nemo.core import NeuralType +from nemo.core import NeuralType, LossType __all__ = ['LossAggregatorNM'] @@ -35,7 +35,7 @@ def input_ports(self): """ input_ports = {} for i in range(self.num_losses): - input_ports["loss_" + str(i + 1)] = NeuralType(None) + input_ports["loss_" + str(i + 1)] = NeuralType() return input_ports @@ -46,7 +46,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, num_inputs=2): # Store number of inputs/losses. diff --git a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py index 3ba4d631f1da..fa5e2dc186d5 100644 --- a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py +++ b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py @@ -18,7 +18,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LossType, LogitsType __all__ = ['JointIntentSlotLoss'] @@ -49,38 +49,18 @@ class JointIntentSlotLoss(LossNM): def input_ports(self): """Returns definitions of module input ports. - intent_logits: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - slot_logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - intents: - 0: AxisType(BatchTag) - - slots: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - """ return { - "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "intents": NeuralType({0: AxisType(BatchTag)}), - "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "intents": NeuralType({0: AxisType(BatchTag)}), + # "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "intent_logits": NeuralType(LogitsType(), ('B', 'D')), + "slot_logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "intents": NeuralType(ChannelType(), tuple('B')), + "slots": NeuralType(ChannelType(), ('B', 'T')), } @property @@ -90,7 +70,8 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + #return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__( self, num_slots, slot_classes_loss_weights=None, intent_classes_loss_weights=None, intent_loss_weight=0.6, diff --git a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py index e5516d9f33c7..485d49345b9b 100644 --- a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py +++ b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py @@ -16,7 +16,7 @@ from nemo.backends.pytorch import LossNM from nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss import SmoothedCrossEntropyLoss -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LossType, LogitsType __all__ = ['MaskedLanguageModelingLossNM'] @@ -32,28 +32,14 @@ class MaskedLanguageModelingLossNM(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - output_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - output_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "output_ids": NeuralType(ChannelType(), ('B', 'T')), + "output_mask": NeuralType(ChannelType(), ('B', 'T')), } @property @@ -63,7 +49,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, label_smoothing=0.0): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py index 0ad66e21106d..292cc77c932b 100644 --- a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py +++ b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py @@ -17,7 +17,7 @@ from nemo.backends.pytorch import LossNM from nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss import SmoothedCrossEntropyLoss from nemo.collections.nlp.utils.common_nlp_utils import mask_padded_tokens -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import AxisType, NeuralType, ChannelType, LogitsType, LossType __all__ = ['PaddedSmoothedCrossEntropyLossNM'] @@ -38,32 +38,20 @@ class PaddedSmoothedCrossEntropyLossNM(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - target_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "target_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "target_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "target_ids": NeuralType(ChannelType(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(None)} + # return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, pad_id, label_smoothing=0, predict_last_k=0): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/losses/qa_squad_loss.py b/nemo/collections/nlp/nm/losses/qa_squad_loss.py index 5f60871d4ebb..acc53066c413 100644 --- a/nemo/collections/nlp/nm/losses/qa_squad_loss.py +++ b/nemo/collections/nlp/nm/losses/qa_squad_loss.py @@ -17,7 +17,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LogitsType, LossType __all__ = ['QuestionAnsweringLoss'] @@ -38,24 +38,14 @@ class QuestionAnsweringLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - start_positions: - 0: AxisType(BatchTag) - - end_positions: - 0: AxisType(BatchTag) """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "start_positions": NeuralType({0: AxisType(BatchTag)}), - "end_positions": NeuralType({0: AxisType(BatchTag)}), + # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "start_positions": NeuralType({0: AxisType(BatchTag)}), + # "end_positions": NeuralType({0: AxisType(BatchTag)}), + "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "start_positions": NeuralType(ChannelType(), tuple('B')), + "end_positions": NeuralType(ChannelType(), tuple('B')), } @property @@ -76,9 +66,12 @@ def output_ports(self): 1: AxisType(TimeTag) """ return { - "loss": NeuralType(None), - "start_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "end_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss": NeuralType(None), + # "start_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "end_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "loss": NeuralType(LossType()), + "start_logits": NeuralType(ChannelType(), ('B', 'T')), + "end_logits": NeuralType(ChannelType(), ('B', 'T')), } def __init__(self): diff --git a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py index c591fc453afb..8f13572479ce 100644 --- a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py +++ b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py @@ -39,7 +39,7 @@ import torch from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import NeuralType, ChannelType, LogitsType, LossType, LabelsType __all__ = ['TRADEMaskedCrossEntropy', 'CrossEntropyLoss3D'] @@ -61,44 +61,29 @@ def input_ports(self): """Returns definitions of module input ports. logits: 4d tensor of logits - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - 3: AxisType(ChannelTag) targets: 3d tensor of labels - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(TimeTag) loss_mask: specifies the words to be considered in the loss calculation - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ return { - "logits": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag), 3: AxisType(ChannelTag)} - ), - "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "logits": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag), 3: AxisType(ChannelTag)} + # ), + # "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "logits": NeuralType(LogitsType(), ('B', 'T', 'D', 'D')), + "targets": NeuralType(ChannelType(), ('B', 'D', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'D')), } @property def output_ports(self): """Returns definitions of module output ports. - - loss: loss value - NeuralType(None) - """ - return {"loss": NeuralType(None)} + # return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self): LossNM.__init__(self) @@ -139,15 +124,18 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), - "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), + # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + "logits": NeuralType(LogitsType(), ('B', 'D', 'D')), + "labels": NeuralType(LabelsType(), ('B', 'D')), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(None)} + # return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, num_classes, **kwargs): LossNM.__init__(self, **kwargs) diff --git a/nemo/collections/nlp/nm/losses/token_classification_loss.py b/nemo/collections/nlp/nm/losses/token_classification_loss.py index 5c3c3adcad22..411264296fe5 100644 --- a/nemo/collections/nlp/nm/losses/token_classification_loss.py +++ b/nemo/collections/nlp/nm/losses/token_classification_loss.py @@ -18,7 +18,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LossType, LabelsType, LogitsType __all__ = ['TokenClassificationLoss'] @@ -38,28 +38,14 @@ class TokenClassificationLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - labels: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - loss_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "labels": NeuralType(LabelsType(), ('B', 'T')), + "loss_mask": NeuralType(ChannelType(), ('B', 'T')), } @property @@ -69,7 +55,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, num_classes, class_weights=None): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py index 1f91576be60a..3d313e5b0dd7 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py @@ -20,7 +20,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import NeuralType, ChannelType __all__ = ['BERT'] @@ -49,40 +49,22 @@ class BERT(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - token_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - attention_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "token_type_ids": NeuralType(ChannelType(), ('B', 'T')), + "attention_mask": NeuralType(ChannelType(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py index 7e0c81c65388..9ca9aabdd6df 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LogitsType __all__ = ['SequenceClassifier'] @@ -41,26 +41,15 @@ class SequenceClassifier(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} @property def output_ports(self): """Returns definitions of module output ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) """ - return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} + # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} + return {"logits": NeuralType(LogitsType(), ('B', 'D'))} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py index 1032a1f2c43d..1c16bc967330 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, RegressionTag, TimeTag +from nemo.core import NeuralType, RegressionValuesType, ChannelType __all__ = ['SequenceRegression'] @@ -39,24 +39,16 @@ class SequenceRegression(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} @property def output_ports(self): """Returns definitions of module output ports. - - preds: - 0: AxisType(RegressionTag) """ - return {"preds": NeuralType({0: AxisType(RegressionTag)})} + # return {"preds": NeuralType({0: AxisType(RegressionTag)})} + return {"preds": NeuralType(RegressionValuesType(), tuple('B'))} def __init__(self, hidden_size, num_layers=2, activation='relu', dropout=0.0, use_transformer_pretrained=True): super().__init__() diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index ba848f247eb3..4fa8d0478e38 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import gelu, transformer_weights_init -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LogitsType __all__ = ['BertTokenClassifier', 'TokenClassifier'] @@ -42,28 +42,16 @@ class BertTokenClassifier(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} @property def output_ports(self): """Returns definitions of module output ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"logits": NeuralType(LogitsType(), ('B', 'T', 'C'))} def __init__( self, @@ -115,28 +103,16 @@ class TokenClassifier(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'C'))} @property def output_ports(self): """Returns definitions of module output ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"logits": NeuralType(LogitsType(), ('B', 'T', 'D'))} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py index b736588a3d33..cffd22c39c94 100644 --- a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py @@ -14,7 +14,7 @@ ) from nemo.collections.nlp.nm.trainables.common.transformer.transformer_modules import TransformerEmbedding from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core.neural_types import * +from nemo.core.neural_types import NeuralType, ChannelType __all__ = ['TransformerEncoderNM', 'TransformerDecoderNM', 'GreedyLanguageGeneratorNM', 'BeamSearchTranslatorNM'] @@ -47,34 +47,21 @@ class TransformerEncoderNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask_src: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(ChannelType(), ('B', 'T')), + "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} def __init__( self, @@ -149,48 +136,24 @@ class TransformerDecoderNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids_tgt: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - hidden_states_src: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - input_mask_src: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - input_mask_tgt: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "input_ids_tgt": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_mask_tgt": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_ids_tgt": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "input_mask_tgt": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids_tgt": NeuralType(ChannelType(), ('B', 'T')), + "hidden_states_src": NeuralType(ChannelType(), ('B', 'T', 'D')), + "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), + "input_mask_tgt": NeuralType(ChannelType(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} def __init__( self, @@ -255,24 +218,16 @@ class GreedyLanguageGeneratorNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"input_ids": NeuralType(ChannelType(), ('B', 'T'))} @property def output_ports(self): """Returns definitions of module output ports. - - output_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"output_ids": NeuralType(ChannelType(), ('B', 'T'))} def __init__(self, decoder, log_softmax, max_seq_length, pad_token, bos_token, eos_token, batch_size=1): super().__init__() @@ -319,34 +274,20 @@ class BeamSearchTranslatorNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states_src: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - input_mask_src: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "hidden_states_src": NeuralType(ChannelType(), ('B', 'T', 'C')), + "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - output_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"output_ids": NeuralType(ChannelType(), ('B', 'T'))} @property def num_weights(self): diff --git a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py index 5a2aa466afe1..9c6dd5c0d2cd 100644 --- a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py +++ b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py @@ -45,7 +45,7 @@ from torch import nn as nn from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import NeuralType, ChannelType, LengthsType __all__ = ['TRADEGenerator'] @@ -56,41 +56,28 @@ def input_ports(self): """Returns definitions of module input ports. encoder_hidden: hidden states of the encoder - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) encoder_outputs: outputs of the encoder - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) input_lens: lengths of the input sequences to encoder - 0: AxisType(BatchTag) src_ids: input sequences to encoder - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) targets: targets for the output of the generator - 0: AxisType(BatchTag) - - 1: AxisType(BatchTag) - - 2: AxisType(TimeTag) """ return { - 'encoder_hidden': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - 'encoder_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - 'input_lens': NeuralType({0: AxisType(BatchTag)}), - 'src_ids': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), + # 'encoder_hidden': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # 'encoder_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # 'input_lens': NeuralType({0: AxisType(BatchTag)}), + # 'src_ids': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), + 'encoder_hidden': NeuralType(ChannelType(), ('B', 'T', 'C')), + 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'C')), + 'input_lens': NeuralType(LengthsType(), tuple('B')), + 'src_ids': NeuralType(ChannelType(), ('B', 'T')), + 'targets': NeuralType(ChannelType(), ('B', 'D', 'T')), + } @property @@ -98,27 +85,19 @@ def output_ports(self): """Returns definitions of module output ports. point_outputs: outputs of the generator - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - 3: AxisType(ChannelTag) gate_outputs: outputs of gating heads - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(ChannelTag) """ + # return { + # 'point_outputs': NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag), 3: AxisType(ChannelTag)} + # ), + # 'gate_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), + # } return { - 'point_outputs': NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag), 3: AxisType(ChannelTag)} - ), - 'gate_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), + 'point_outputs': NeuralType(ChannelType(), ('B', 'T', 'D', 'D')), + 'gate_outputs': NeuralType(ChannelType(), ('B', 'D', 'D')) } def __init__(self, vocab, embeddings, hid_size, dropout, slots, nb_gate, teacher_forcing=0.5): diff --git a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py index b8707646f746..bf5c88263b48 100644 --- a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py +++ b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core import NeuralType, ChannelType, LogitsType __all__ = ['JointIntentSlotClassifier'] @@ -39,15 +39,9 @@ class JointIntentSlotClassifier(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'C'))} @property def output_ports(self): @@ -66,8 +60,10 @@ def output_ports(self): 2: AxisType(ChannelTag) """ return { - "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), + # "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + "intent_logits": NeuralType(LogitsType(), ('B', 'D')), + "slot_logits": NeuralType(LogitsType(), ('B', 'D')) } def __init__(self, hidden_size, num_intents, num_slots, dropout=0.0, use_transformer_pretrained=True, **kwargs): diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 37f35867a159..3508ea224337 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -28,6 +28,7 @@ 'LabelsType', 'LossType', 'RegressionValuesType', + 'CategoricalValuesType', 'PredictionsType', 'LogprobsType', 'LengthsType', @@ -171,3 +172,7 @@ def __str__(self): class RegressionValuesType(PredictionsType): def __str__(self): return "regression values type" + +class CategoricalValuesType(PredictionsType): + def __str__(self): + return "regression values type" diff --git a/tests/core/test_deploy_export.py b/tests/core/test_deploy_export.py index 3d29c166ea6b..04eed27d48e4 100644 --- a/tests/core/test_deploy_export.py +++ b/tests/core/test_deploy_export.py @@ -1,181 +1,181 @@ -# # ! /usr/bin/python -# # -*- coding: utf-8 -*- -# -# # Copyright 2019 NVIDIA. All Rights Reserved. -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License. -# # ============================================================================= -# -# import os -# from pathlib import Path -# -# # git clone git@github.com:microsoft/onnxruntime.git -# # cd onnxruntime -# # ./build.sh --update --build --config RelWithDebInfo --build_shared_lib --parallel --use_cuda \ -# # --cudnn_home /usr/lib/x86_64-linux-gnu --cuda_home /usr/local/cuda --enable_pybind --build_wheel -# # pip install --upgrade ./build/Linux/RelWithDebInfo/dist/onnxruntime_gpu-1.1.0-cp37-cp37m-linux_x86_64.whl -# import onnxruntime as ort -# import torch -# from ruamel.yaml import YAML -# -# import nemo -# import nemo.collections.asr as nemo_asr -# import nemo.collections.nlp as nemo_nlp -# import nemo.collections.nlp.nm.trainables.common.token_classification_nm -# from tests.common_setup import NeMoUnitTest -# -# -# class TestDeployExport(NeMoUnitTest): -# def setUp(self): -# """ Setups neural factory so it will use GPU instead of CPU. """ -# NeMoUnitTest.setUp(self) -# -# # Perform computations on GPU. -# self.nf._placement = nemo.core.DeviceType.GPU -# -# def __test_export_route(self, module, out_name, mode, input_example=None): -# out = Path(out_name) -# if out.exists(): -# os.remove(out) -# -# outputs_fwd = ( -# (module.forward(*input_example) if isinstance(input_example, tuple) else module.forward(input_example)) -# if input_example is not None -# else None -# ) -# self.nf.deployment_export( -# module=module, output=out_name, input_example=input_example, d_format=mode, output_example=outputs_fwd -# ) -# -# tol = 2.0e-3 -# self.assertTrue(out.exists()) -# if mode == nemo.core.DeploymentFormat.ONNX: -# # Must recompute beause *module* might be different now -# outputs_fwd = ( -# module.forward(*input_example) if isinstance(input_example, tuple) else module.forward(input_example) -# ) -# sess_options = ort.SessionOptions() -# sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED -# ort_session = ort.InferenceSession(out_name, sess_options) -# inputs = dict() -# input_names = list(module.input_ports) -# for i in range(len(input_names)): -# input_name = ( -# "encoded_lengths" -# if type(module).__name__ == "JasperEncoder" and input_names[i] == "length" -# else input_names[i] -# ) -# inputs[input_name] = ( -# input_example[i].cpu().numpy() if isinstance(input_example, tuple) else input_example.cpu().numpy() -# ) -# outputs_scr = ort_session.run(None, inputs) -# outputs_scr = torch.from_numpy(outputs_scr[0]).cuda() -# outputs_fwd = outputs_fwd[0] if isinstance(outputs_fwd, tuple) else outputs_fwd -# tol = 5.0e-4 -# elif mode == nemo.core.DeploymentFormat.TORCHSCRIPT: -# scr = torch.jit.load(out_name) -# if isinstance(module, nemo.backends.pytorch.tutorials.TaylorNet): -# input_example = torch.randn(4, 1).cuda() -# outputs_fwd = module.forward(input_example) -# outputs_scr = ( -# scr.forward(*input_example) if isinstance(input_example, tuple) else scr.forward(input_example) -# ) -# elif mode == nemo.core.DeploymentFormat.PYTORCH: -# module.load_state_dict(torch.load(out_name)) -# module.eval() -# outputs_scr = module.forward(*input_example) -# -# self.assertLess((outputs_scr - outputs_fwd).norm(p=2), tol) -# -# if out.exists(): -# os.remove(out) -# -# def test_simple_module_export(self): -# simplest_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) -# self.__test_export_route( -# module=simplest_module, -# out_name="simple.pt", -# mode=nemo.core.DeploymentFormat.TORCHSCRIPT, -# input_example=None, -# ) -# -# def test_TokenClassifier_module_export(self): -# t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( -# hidden_size=512, num_classes=16, use_transformer_pretrained=False -# ) -# self.__test_export_route( -# module=t_class, -# out_name="t_class.pt", -# mode=nemo.core.DeploymentFormat.TORCHSCRIPT, -# input_example=torch.randn(16, 16, 512).cuda(), -# ) -# -# def test_TokenClassifier_module_onnx_export(self): -# t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( -# hidden_size=512, num_classes=16, use_transformer_pretrained=False -# ) -# self.__test_export_route( -# module=t_class, -# out_name="t_class.onnx", -# mode=nemo.core.DeploymentFormat.ONNX, -# input_example=torch.randn(16, 16, 512).cuda(), -# ) -# -# def test_jasper_decoder_export_ts(self): -# j_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=33) -# self.__test_export_route( -# module=j_decoder, -# out_name="j_decoder.ts", -# mode=nemo.core.DeploymentFormat.TORCHSCRIPT, -# input_example=torch.randn(34, 1024, 1).cuda(), -# ) -# -# def test_hf_bert_ts(self): -# bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") -# input_example = ( -# torch.randint(low=0, high=16, size=(2, 16)).cuda(), -# torch.randint(low=0, high=1, size=(2, 16)).cuda(), -# torch.randint(low=0, high=1, size=(2, 16)).cuda(), -# ) -# self.__test_export_route( -# module=bert, out_name="bert.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=input_example -# ) -# -# def test_hf_bert_pt(self): -# bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") -# input_example = ( -# torch.randint(low=0, high=16, size=(2, 16)).cuda(), -# torch.randint(low=0, high=1, size=(2, 16)).cuda(), -# torch.randint(low=0, high=1, size=(2, 16)).cuda(), -# ) -# self.__test_export_route( -# module=bert, out_name="bert.pt", mode=nemo.core.DeploymentFormat.PYTORCH, input_example=input_example, -# ) -# -# def test_jasper_encoder_to_onnx(self): -# with open("tests/data/jasper_smaller.yaml") as file: -# yaml = YAML(typ="safe") -# jasper_model_definition = yaml.load(file) -# -# jasper_encoder = nemo_asr.JasperEncoder( -# conv_mask=False, -# feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], -# **jasper_model_definition['JasperEncoder'] -# ) -# -# self.__test_export_route( -# module=jasper_encoder, -# out_name="jasper_encoder.onnx", -# mode=nemo.core.DeploymentFormat.ONNX, -# input_example=(torch.randn(16, 64, 256).cuda(), torch.randn(256).cuda()), -# ) +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2019 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import os +from pathlib import Path + +# git clone git@github.com:microsoft/onnxruntime.git +# cd onnxruntime +# ./build.sh --update --build --config RelWithDebInfo --build_shared_lib --parallel --use_cuda \ +# --cudnn_home /usr/lib/x86_64-linux-gnu --cuda_home /usr/local/cuda --enable_pybind --build_wheel +# pip install --upgrade ./build/Linux/RelWithDebInfo/dist/onnxruntime_gpu-1.1.0-cp37-cp37m-linux_x86_64.whl +import onnxruntime as ort +import torch +from ruamel.yaml import YAML + +import nemo +import nemo.collections.asr as nemo_asr +import nemo.collections.nlp as nemo_nlp +import nemo.collections.nlp.nm.trainables.common.token_classification_nm +from tests.common_setup import NeMoUnitTest + + +class TestDeployExport(NeMoUnitTest): + def setUp(self): + """ Setups neural factory so it will use GPU instead of CPU. """ + NeMoUnitTest.setUp(self) + + # Perform computations on GPU. + self.nf._placement = nemo.core.DeviceType.GPU + + def __test_export_route(self, module, out_name, mode, input_example=None): + out = Path(out_name) + if out.exists(): + os.remove(out) + + outputs_fwd = ( + (module.forward(*input_example) if isinstance(input_example, tuple) else module.forward(input_example)) + if input_example is not None + else None + ) + self.nf.deployment_export( + module=module, output=out_name, input_example=input_example, d_format=mode, output_example=outputs_fwd + ) + + tol = 2.0e-3 + self.assertTrue(out.exists()) + if mode == nemo.core.DeploymentFormat.ONNX: + # Must recompute beause *module* might be different now + outputs_fwd = ( + module.forward(*input_example) if isinstance(input_example, tuple) else module.forward(input_example) + ) + sess_options = ort.SessionOptions() + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + ort_session = ort.InferenceSession(out_name, sess_options) + inputs = dict() + input_names = list(module.input_ports) + for i in range(len(input_names)): + input_name = ( + "encoded_lengths" + if type(module).__name__ == "JasperEncoder" and input_names[i] == "length" + else input_names[i] + ) + inputs[input_name] = ( + input_example[i].cpu().numpy() if isinstance(input_example, tuple) else input_example.cpu().numpy() + ) + outputs_scr = ort_session.run(None, inputs) + outputs_scr = torch.from_numpy(outputs_scr[0]).cuda() + outputs_fwd = outputs_fwd[0] if isinstance(outputs_fwd, tuple) else outputs_fwd + tol = 5.0e-4 + elif mode == nemo.core.DeploymentFormat.TORCHSCRIPT: + scr = torch.jit.load(out_name) + if isinstance(module, nemo.backends.pytorch.tutorials.TaylorNet): + input_example = torch.randn(4, 1).cuda() + outputs_fwd = module.forward(input_example) + outputs_scr = ( + scr.forward(*input_example) if isinstance(input_example, tuple) else scr.forward(input_example) + ) + elif mode == nemo.core.DeploymentFormat.PYTORCH: + module.load_state_dict(torch.load(out_name)) + module.eval() + outputs_scr = module.forward(*input_example) + + self.assertLess((outputs_scr - outputs_fwd).norm(p=2), tol) + + if out.exists(): + os.remove(out) + + def test_simple_module_export(self): + simplest_module = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) + self.__test_export_route( + module=simplest_module, + out_name="simple.pt", + mode=nemo.core.DeploymentFormat.TORCHSCRIPT, + input_example=None, + ) + + def test_TokenClassifier_module_export(self): + t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( + hidden_size=512, num_classes=16, use_transformer_pretrained=False + ) + self.__test_export_route( + module=t_class, + out_name="t_class.pt", + mode=nemo.core.DeploymentFormat.TORCHSCRIPT, + input_example=torch.randn(16, 16, 512).cuda(), + ) + + def test_TokenClassifier_module_onnx_export(self): + t_class = nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier( + hidden_size=512, num_classes=16, use_transformer_pretrained=False + ) + self.__test_export_route( + module=t_class, + out_name="t_class.onnx", + mode=nemo.core.DeploymentFormat.ONNX, + input_example=torch.randn(16, 16, 512).cuda(), + ) + + def test_jasper_decoder_export_ts(self): + j_decoder = nemo_asr.JasperDecoderForCTC(feat_in=1024, num_classes=33) + self.__test_export_route( + module=j_decoder, + out_name="j_decoder.ts", + mode=nemo.core.DeploymentFormat.TORCHSCRIPT, + input_example=torch.randn(34, 1024, 1).cuda(), + ) + + def test_hf_bert_ts(self): + bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") + input_example = ( + torch.randint(low=0, high=16, size=(2, 16)).cuda(), + torch.randint(low=0, high=1, size=(2, 16)).cuda(), + torch.randint(low=0, high=1, size=(2, 16)).cuda(), + ) + self.__test_export_route( + module=bert, out_name="bert.ts", mode=nemo.core.DeploymentFormat.TORCHSCRIPT, input_example=input_example + ) + + def test_hf_bert_pt(self): + bert = nemo.collections.nlp.nm.trainables.common.huggingface.BERT(pretrained_model_name="bert-base-uncased") + input_example = ( + torch.randint(low=0, high=16, size=(2, 16)).cuda(), + torch.randint(low=0, high=1, size=(2, 16)).cuda(), + torch.randint(low=0, high=1, size=(2, 16)).cuda(), + ) + self.__test_export_route( + module=bert, out_name="bert.pt", mode=nemo.core.DeploymentFormat.PYTORCH, input_example=input_example, + ) + + def test_jasper_encoder_to_onnx(self): + with open("tests/data/jasper_smaller.yaml") as file: + yaml = YAML(typ="safe") + jasper_model_definition = yaml.load(file) + + jasper_encoder = nemo_asr.JasperEncoder( + conv_mask=False, + feat_in=jasper_model_definition['AudioToMelSpectrogramPreprocessor']['features'], + **jasper_model_definition['JasperEncoder'] + ) + + self.__test_export_route( + module=jasper_encoder, + out_name="jasper_encoder.onnx", + mode=nemo.core.DeploymentFormat.ONNX, + input_example=(torch.randn(16, 64, 256).cuda(), torch.randn(256).cuda()), + ) From 8e3b2c2c982dbad16674f9c214a14d6a3ef666d7 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Thu, 6 Feb 2020 16:55:14 -0800 Subject: [PATCH 10/70] fix style Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/data_layers/glue_benchmark_datalayer.py | 2 +- .../nlp/nm/data_layers/joint_intent_slot_datalayer.py | 4 ++-- nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py | 2 +- .../nlp/nm/data_layers/lm_transformer_datalayer.py | 2 +- .../nlp/nm/data_layers/machine_translation_datalayer.py | 2 +- .../nm/data_layers/punctuation_capitalization_datalayer.py | 2 +- nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py | 3 +-- .../nlp/nm/data_layers/state_tracking_trade_datalayer.py | 2 +- .../nlp/nm/data_layers/text_classification_datalayer.py | 2 +- .../nlp/nm/data_layers/token_classification_datalayer.py | 2 +- nemo/collections/nlp/nm/losses/aggregator_loss.py | 2 +- nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py | 4 ++-- .../nlp/nm/losses/masked_language_modeling_loss.py | 2 +- .../nlp/nm/losses/padded_smoothed_cross_entropy_loss.py | 2 +- nemo/collections/nlp/nm/losses/qa_squad_loss.py | 2 +- nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py | 2 +- nemo/collections/nlp/nm/losses/token_classification_loss.py | 2 +- .../nlp/nm/trainables/common/huggingface/bert_nm.py | 2 +- .../nlp/nm/trainables/common/sequence_classification_nm.py | 2 +- .../nlp/nm/trainables/common/sequence_regression_nm.py | 2 +- .../nlp/nm/trainables/common/token_classification_nm.py | 2 +- .../nlp/nm/trainables/common/transformer/transformer_nm.py | 2 +- .../dialogue_state_tracking/state_tracking_trade_nm.py | 5 ++--- .../nm/trainables/joint_intent_slot/joint_intent_slot_nm.py | 4 ++-- nemo/core/neural_types/elements.py | 1 + 25 files changed, 29 insertions(+), 30 deletions(-) diff --git a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py index 56dea4219240..d8426e9b425d 100644 --- a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import GLUEDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, RegressionValuesType, ChannelType, CategoricalValuesType +from nemo.core import CategoricalValuesType, ChannelType, NeuralType, RegressionValuesType __all__ = ['GlueClassificationDataLayer', 'GlueRegressionDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py index bb95726a2e78..b5b04f6ab299 100644 --- a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertJointIntentSlotDataset, BertJointIntentSlotInferDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType +from nemo.core import ChannelType, NeuralType __all__ = ['BertJointIntentSlotDataLayer', 'BertJointIntentSlotInferDataLayer'] @@ -122,7 +122,7 @@ def output_ports(self): "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), "input_mask": NeuralType(ChannelType(), ('B', 'T')), "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')) + "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), } def __init__(self, queries, tokenizer, max_seq_length, batch_size=1, dataset_type=BertJointIntentSlotInferDataset): diff --git a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py index e96be1a1e788..c5e02c7ca38f 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch import DataLayerNM from nemo.collections.nlp.data import BertPretrainingDataset, BertPretrainingPreprocessedDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['BertPretrainingDataLayer', 'BertPretrainingPreprocessedDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py index 266fb4f2ffc7..c87fbd9c500a 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import LanguageModelingDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['LanguageModelingDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py index 7f13898ea1fc..6fca7e3790c2 100644 --- a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py @@ -20,7 +20,7 @@ import nemo from nemo.collections.nlp.data import TranslationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['TranslationDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py index 7b4fa9b77133..84d281e78737 100644 --- a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertPunctuationCapitalizationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['PunctuationCapitalizationDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py index 245e05eb309a..1f0b32ca53f8 100644 --- a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import SquadDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['BertQuestionAnsweringDataLayer'] @@ -62,7 +62,6 @@ def output_ports(self): "start_positions": NeuralType(ChannelType(), tuple('B')), "end_positions": NeuralType(ChannelType(), tuple('B')), "unique_ids": NeuralType(ChannelType(), tuple('B')), - } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index 6845c47de4bc..da51068b8519 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -43,7 +43,7 @@ import nemo from nemo.collections.nlp.data.datasets import MultiWOZDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core.neural_types import NeuralType, ChannelType, LabelsType +from nemo.core.neural_types import ChannelType, LabelsType, NeuralType __all__ = ['MultiWOZDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py index f1f408580069..935e57fe53bf 100644 --- a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertTextClassificationDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['BertSentenceClassificationDataLayer'] diff --git a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py index 3744f3b8682b..f4ad2e613da5 100644 --- a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py @@ -16,7 +16,7 @@ from nemo.collections.nlp.data import BertTokenClassificationDataset, BertTokenClassificationInferDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core import NeuralType, ChannelType, LabelsType +from nemo.core import ChannelType, LabelsType, NeuralType __all__ = ['BertTokenClassificationDataLayer', 'BertTokenClassificationInferDataLayer'] diff --git a/nemo/collections/nlp/nm/losses/aggregator_loss.py b/nemo/collections/nlp/nm/losses/aggregator_loss.py index 0720d0409fd8..62b775f02927 100644 --- a/nemo/collections/nlp/nm/losses/aggregator_loss.py +++ b/nemo/collections/nlp/nm/losses/aggregator_loss.py @@ -15,7 +15,7 @@ # ============================================================================= from nemo.backends.pytorch import LossNM -from nemo.core import NeuralType, LossType +from nemo.core import LossType, NeuralType __all__ = ['LossAggregatorNM'] diff --git a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py index fa5e2dc186d5..5ef07b54e347 100644 --- a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py +++ b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py @@ -18,7 +18,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import NeuralType, ChannelType, LossType, LogitsType +from nemo.core import ChannelType, LogitsType, LossType, NeuralType __all__ = ['JointIntentSlotLoss'] @@ -70,7 +70,7 @@ def output_ports(self): loss: NeuralType(None) """ - #return {"loss": NeuralType(None)} + # return {"loss": NeuralType(None)} return {"loss": NeuralType(LossType())} def __init__( diff --git a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py index 485d49345b9b..6e157a40c511 100644 --- a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py +++ b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py @@ -16,7 +16,7 @@ from nemo.backends.pytorch import LossNM from nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss import SmoothedCrossEntropyLoss -from nemo.core import NeuralType, ChannelType, LossType, LogitsType +from nemo.core import ChannelType, LogitsType, LossType, NeuralType __all__ = ['MaskedLanguageModelingLossNM'] diff --git a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py index 292cc77c932b..b56717af885d 100644 --- a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py +++ b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py @@ -17,7 +17,7 @@ from nemo.backends.pytorch import LossNM from nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss import SmoothedCrossEntropyLoss from nemo.collections.nlp.utils.common_nlp_utils import mask_padded_tokens -from nemo.core import AxisType, NeuralType, ChannelType, LogitsType, LossType +from nemo.core import AxisType, ChannelType, LogitsType, LossType, NeuralType __all__ = ['PaddedSmoothedCrossEntropyLossNM'] diff --git a/nemo/collections/nlp/nm/losses/qa_squad_loss.py b/nemo/collections/nlp/nm/losses/qa_squad_loss.py index acc53066c413..e1718592a058 100644 --- a/nemo/collections/nlp/nm/losses/qa_squad_loss.py +++ b/nemo/collections/nlp/nm/losses/qa_squad_loss.py @@ -17,7 +17,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import NeuralType, ChannelType, LogitsType, LossType +from nemo.core import ChannelType, LogitsType, LossType, NeuralType __all__ = ['QuestionAnsweringLoss'] diff --git a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py index 8f13572479ce..662de183a183 100644 --- a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py +++ b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py @@ -39,7 +39,7 @@ import torch from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import NeuralType, ChannelType, LogitsType, LossType, LabelsType +from nemo.core.neural_types import ChannelType, LabelsType, LogitsType, LossType, NeuralType __all__ = ['TRADEMaskedCrossEntropy', 'CrossEntropyLoss3D'] diff --git a/nemo/collections/nlp/nm/losses/token_classification_loss.py b/nemo/collections/nlp/nm/losses/token_classification_loss.py index 411264296fe5..ccdbe5100d72 100644 --- a/nemo/collections/nlp/nm/losses/token_classification_loss.py +++ b/nemo/collections/nlp/nm/losses/token_classification_loss.py @@ -18,7 +18,7 @@ from torch import nn from nemo.backends.pytorch import LossNM -from nemo.core import NeuralType, ChannelType, LossType, LabelsType, LogitsType +from nemo.core import ChannelType, LabelsType, LogitsType, LossType, NeuralType __all__ = ['TokenClassificationLoss'] diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py index 3d313e5b0dd7..40dd18c7bbe2 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py @@ -20,7 +20,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import NeuralType, ChannelType +from nemo.core.neural_types import ChannelType, NeuralType __all__ = ['BERT'] diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py index 9ca9aabdd6df..5781ecb0c064 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import NeuralType, ChannelType, LogitsType +from nemo.core import ChannelType, LogitsType, NeuralType __all__ = ['SequenceClassifier'] diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py index 1c16bc967330..9b7cc38ad4fd 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import NeuralType, RegressionValuesType, ChannelType +from nemo.core import ChannelType, NeuralType, RegressionValuesType __all__ = ['SequenceRegression'] diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index 4fa8d0478e38..045fa55a7099 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import gelu, transformer_weights_init -from nemo.core import NeuralType, ChannelType, LogitsType +from nemo.core import ChannelType, LogitsType, NeuralType __all__ = ['BertTokenClassifier', 'TokenClassifier'] diff --git a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py index cffd22c39c94..0822d769d246 100644 --- a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py @@ -14,7 +14,7 @@ ) from nemo.collections.nlp.nm.trainables.common.transformer.transformer_modules import TransformerEmbedding from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core.neural_types import NeuralType, ChannelType +from nemo.core.neural_types import ChannelType, NeuralType __all__ = ['TransformerEncoderNM', 'TransformerDecoderNM', 'GreedyLanguageGeneratorNM', 'BeamSearchTranslatorNM'] diff --git a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py index 9c6dd5c0d2cd..0ac416d4ab73 100644 --- a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py +++ b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py @@ -45,7 +45,7 @@ from torch import nn as nn from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import NeuralType, ChannelType, LengthsType +from nemo.core.neural_types import ChannelType, LengthsType, NeuralType __all__ = ['TRADEGenerator'] @@ -77,7 +77,6 @@ def input_ports(self): 'input_lens': NeuralType(LengthsType(), tuple('B')), 'src_ids': NeuralType(ChannelType(), ('B', 'T')), 'targets': NeuralType(ChannelType(), ('B', 'D', 'T')), - } @property @@ -97,7 +96,7 @@ def output_ports(self): # } return { 'point_outputs': NeuralType(ChannelType(), ('B', 'T', 'D', 'D')), - 'gate_outputs': NeuralType(ChannelType(), ('B', 'D', 'D')) + 'gate_outputs': NeuralType(ChannelType(), ('B', 'D', 'D')), } def __init__(self, vocab, embeddings, hid_size, dropout, slots, nb_gate, teacher_forcing=0.5): diff --git a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py index bf5c88263b48..faa273919d15 100644 --- a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py +++ b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py @@ -18,7 +18,7 @@ from nemo.backends.pytorch import MultiLayerPerceptron, TrainableNM from nemo.collections.nlp.nm.trainables.common.transformer.transformer_utils import transformer_weights_init -from nemo.core import NeuralType, ChannelType, LogitsType +from nemo.core import ChannelType, LogitsType, NeuralType __all__ = ['JointIntentSlotClassifier'] @@ -63,7 +63,7 @@ def output_ports(self): # "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), "intent_logits": NeuralType(LogitsType(), ('B', 'D')), - "slot_logits": NeuralType(LogitsType(), ('B', 'D')) + "slot_logits": NeuralType(LogitsType(), ('B', 'D')), } def __init__(self, hidden_size, num_intents, num_slots, dropout=0.0, use_transformer_pretrained=True, **kwargs): diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 3508ea224337..bde89e9006c5 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -173,6 +173,7 @@ class RegressionValuesType(PredictionsType): def __str__(self): return "regression values type" + class CategoricalValuesType(PredictionsType): def __str__(self): return "regression values type" From 80b5bc296860f56467a968a731ff67c8de4a90ad Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 09:59:50 -0800 Subject: [PATCH 11/70] fix tts collection Signed-off-by: Oleksii Kuchaiev --- nemo/collections/tts/data_layers.py | 16 +- nemo/collections/tts/tacotron2_modules.py | 311 ++++++---------------- nemo/collections/tts/waveglow_modules.py | 90 ++----- nemo/core/neural_types/elements.py | 14 +- 4 files changed, 130 insertions(+), 301 deletions(-) diff --git a/nemo/collections/tts/data_layers.py b/nemo/collections/tts/data_layers.py index cad859fb10cb..12639eaca426 100644 --- a/nemo/collections/tts/data_layers.py +++ b/nemo/collections/tts/data_layers.py @@ -5,7 +5,7 @@ from .parts.datasets import AudioOnlyDataset from nemo.backends.pytorch.nm import DataLayerNM from nemo.core import DeviceType -from nemo.core.neural_types import * +from nemo.core.neural_types import NeuralType, AudioSignal, LengthsType class AudioDataLayer(DataLayerNM): @@ -48,18 +48,12 @@ class AudioDataLayer(DataLayerNM): @property def output_ports(self): """Returns definitions of module output ports. - - audio_signal: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - a_sig_length: - 0: AxisType(BatchTag) """ return { - "audio_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "a_sig_length": NeuralType({0: AxisType(BatchTag)}), + # "audio_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "a_sig_length": NeuralType({0: AxisType(BatchTag)}), + "audio_signal": NeuralType(AudioSignal(), ('B', 'T')), + "a_sig_length": NeuralType(LengthsType(), tuple('B')), } def __init__( diff --git a/nemo/collections/tts/tacotron2_modules.py b/nemo/collections/tts/tacotron2_modules.py index 0613311d3dc4..697c1aa3083b 100644 --- a/nemo/collections/tts/tacotron2_modules.py +++ b/nemo/collections/tts/tacotron2_modules.py @@ -35,29 +35,19 @@ class TextEmbedding(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - char_phone - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"char_phone": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"char_phone": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"char_phone": NeuralType(ChannelType(), ('B', 'T'))} @property def output_ports(self): """Returns definitions of module output ports. - - char_phone_embeddings: - 0: AxisType(BatchTag) - - 1: AxisType(EmbeddedTextTag) - - 2: AxisType(TimeTag)}) """ return { - "char_phone_embeddings": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} - ) + # "char_phone_embeddings": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} + # ) + "char_phone_embeddings": NeuralType(EmbeddedTextType(), ('B', 'D', 'T')) } def __init__(self, n_symbols, symbols_embedding_dim: int = 512): @@ -87,39 +77,25 @@ class Tacotron2Encoder(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - char_phone_embeddings: - 0: AxisType(BatchTag) - - 1: AxisType(EmbeddedTextTag) - - 2: AxisType(TimeTag) - - embedding_length: - 0: AxisType(BatchTag) """ return { - "char_phone_embeddings": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} - ), - "embedding_length": NeuralType({0: AxisType(BatchTag)}), + # "char_phone_embeddings": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} + # ), + # "embedding_length": NeuralType({0: AxisType(BatchTag)}), + "char_phone_embeddings": NeuralType(EmbeddedTextType(), ('B', 'D', 'T')), + "embedding_length": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - char_phone_embeddings: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(EncodedRepresentationTag)}) """ return { - "char_phone_encoded": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} - ) + # "char_phone_encoded": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} + # ) + "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')) } def __init__( @@ -179,63 +155,33 @@ class Tacotron2Decoder(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - char_phone_encoded: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(EncodedRepresentationTag) - - encoded_length: - 0: AxisType(BatchTag) - - mel_target: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) """ return { - "char_phone_encoded": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} - ), - "encoded_length": NeuralType({0: AxisType(BatchTag)}), - "mel_target": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), + # "char_phone_encoded": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} + # ), + # "encoded_length": NeuralType({0: AxisType(BatchTag)}), + # "mel_target": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), + "encoded_length": NeuralType(LengthsType(), tuple('B')), + "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) } @property def output_ports(self): """Returns definitions of module output ports. - - mel_output: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - gate_output: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - alignments: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(TimeTag) """ return { - "mel_output": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), + # "mel_output": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), + "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "gate_output": NeuralType(ChannelType(), ('B', 'T')), + "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')) } def __init__( @@ -326,57 +272,31 @@ class Tacotron2DecoderInfer(Tacotron2Decoder): @property def input_ports(self): """Returns definitions of module input ports. - - char_phone_encoded: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(EncodedRepresentationTag) - - encoded_length: - 0: AxisType(BatchTag) """ return { - "char_phone_encoded": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} - ), - "encoded_length": NeuralType({0: AxisType(BatchTag)}), + # "char_phone_encoded": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} + # ), + # "encoded_length": NeuralType({0: AxisType(BatchTag)}), + "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), + "encoded_length": NeuralType(LengthsType(), tuple('B')) } @property def output_ports(self): """Returns definitions of module output ports. - - mel_output: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - gate_output: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - alignments: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(TimeTag) - - mel_len: - 0: AxisType(BatchTag) """ return { - "mel_output": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), - "mel_len": NeuralType({0: AxisType(BatchTag)}), + # "mel_output": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), + # "mel_len": NeuralType({0: AxisType(BatchTag)}), + "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "gate_output": NeuralType(ChannelType(), ('B', 'T')), + "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')), + "mel_len": NeuralType(LengthsType(), tuple('B')), } def __str__(self): @@ -411,35 +331,23 @@ class Tacotron2Postnet(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - mel_input: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) """ return { - "mel_input": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ) + # "mel_input": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ) + "mel_input": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) } @property def output_ports(self): """Returns definitions of module output ports. - - mel_output: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) """ return { - "mel_output": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), + # "mel_output": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) } def __init__( @@ -482,68 +390,35 @@ class Tacotron2Loss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - mel_out: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - mel_out_postnet: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - gate_out: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - mel_target: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - gate_target: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - target_len: - 0: AxisType(BatchTag) - - seq_len: - 0: AxisType(BatchTag) """ return { - "mel_out": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "mel_out_postnet": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "gate_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "mel_target": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "target_len": NeuralType({0: AxisType(BatchTag)}), - "seq_len": NeuralType({0: AxisType(BatchTag)}), + # "mel_out": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "mel_out_postnet": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "gate_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "mel_target": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "target_len": NeuralType({0: AxisType(BatchTag)}), + # "seq_len": NeuralType({0: AxisType(BatchTag)}), + "mel_out": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "mel_out_postnet": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "gate_out": NeuralType(ChannelType(), ('B', 'T')), + "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "gate_target": NeuralType(ChannelType(), ('B', 'T')), + "target_len": NeuralType(LengthsType(), tuple('B')), + "seq_len": NeuralType(LengthsType(), tuple('B')), } @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, pad_value: float = -11.52): super().__init__() @@ -595,34 +470,22 @@ class MakeGate(NonTrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - target_len: - 0: AxisType(BatchTag) - - mel_target: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) """ return { - "target_len": NeuralType({0: AxisType(BatchTag)}), - "mel_target": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), + # "target_len": NeuralType({0: AxisType(BatchTag)}), + # "mel_target": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + "target_len": NeuralType(LengthsType(), tuple('B')), + "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - gate_target: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"gate_target": NeuralType(ChannelType(), ('B', 'T'))} def forward(self, target_len, mel_target): max_len = mel_target.shape[2] diff --git a/nemo/collections/tts/waveglow_modules.py b/nemo/collections/tts/waveglow_modules.py index 5e13ae73faf9..703b0e8f3458 100644 --- a/nemo/collections/tts/waveglow_modules.py +++ b/nemo/collections/tts/waveglow_modules.py @@ -41,47 +41,28 @@ class WaveGlowNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - mel_spectrogram: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) - - audio: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ return { - "mel_spectrogram": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ), - "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "mel_spectrogram": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ), + # "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "mel_spectrogram": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "audio": NeuralType(AudioSignal(), ('B', 'T')), } @property def output_ports(self): """Returns definitions of module output ports. - - audio: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - log_s_list: - List? - - log_det_W_list: - List? - """ # TODO @blisc: please take a look at those definitions return { - "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "log_s_list": NeuralType(), - "log_det_W_list": NeuralType(), + # "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "log_s_list": NeuralType(), + # "log_det_W_list": NeuralType(), + "audio": NeuralType(AudioSignal(), ('B', 'T')), + "log_s_list": NeuralType(ChannelType()), + "log_det_W_list": NeuralType(ChannelType()), } def __init__( @@ -157,30 +138,20 @@ class WaveGlowInferNM(WaveGlowNM): @property def input_ports(self): """Returns definitions of module input ports. - - mel_spectrogram: - 0: AxisType(BatchTag) - - 1: AxisType(MelSpectrogramSignalTag) - - 2: AxisType(TimeTag) """ return { - "mel_spectrogram": NeuralType( - {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} - ) + # "mel_spectrogram": NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} + # ) + "mel_spectrogram": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) } @property def output_ports(self): """Returns definitions of module output ports. - - audio: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + #return {"audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + return {"audio": NeuralType(AudioSignal(), ('B', 'T'))} def __str__(self): return "WaveGlowNM" @@ -256,33 +227,22 @@ class WaveGlowLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - z: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - log_s_list: - List? - - log_det_W_list: - List? """ # TODO @blisc: please take a look at those definitions return { - "z": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "log_s_list": NeuralType(), - "log_det_W_list": NeuralType(), + # "z": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "log_s_list": NeuralType(), + # "log_det_W_list": NeuralType(), + "z": NeuralType(AudioSignal(), ('B', 'T')), + "log_s_list": NeuralType(ChannelType()), + "log_det_W_list": NeuralType(ChannelType()), } @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, sigma: float = 1.0): super().__init__() diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index bde89e9006c5..47bf93896dc8 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -32,6 +32,8 @@ 'PredictionsType', 'LogprobsType', 'LengthsType', + 'EmbeddedTextType', + 'EncodedRepresentation' ] import abc from abc import ABC, abstractmethod @@ -106,6 +108,11 @@ def __str__(self): return "convolutional channel value" +class EmbeddedTextType(ChannelType): + def __str__(self): + return "text embedding" + + class LogitsType(ElementType): def __str__(self): return "neural type representing logits" @@ -131,7 +138,12 @@ def __str__(self): return "neural type representing loss value" -class AcousticEncodedRepresentation(ChannelType): +class EncodedRepresentation(ChannelType): + def __str__(self): + return "encoded representation, for example, encoder's output" + + +class AcousticEncodedRepresentation(EncodedRepresentation): def __str__(self): return "encoded representation returned by the acoustic encoder model" From 24b2ca4277ed61e17236335068fde1ac7a446856 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 10:03:32 -0800 Subject: [PATCH 12/70] fix code style Signed-off-by: Oleksii Kuchaiev --- nemo/collections/tts/data_layers.py | 2 +- nemo/collections/tts/tacotron2_modules.py | 8 ++++---- nemo/collections/tts/waveglow_modules.py | 2 +- nemo/core/neural_types/elements.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nemo/collections/tts/data_layers.py b/nemo/collections/tts/data_layers.py index 12639eaca426..89344ec85583 100644 --- a/nemo/collections/tts/data_layers.py +++ b/nemo/collections/tts/data_layers.py @@ -5,7 +5,7 @@ from .parts.datasets import AudioOnlyDataset from nemo.backends.pytorch.nm import DataLayerNM from nemo.core import DeviceType -from nemo.core.neural_types import NeuralType, AudioSignal, LengthsType +from nemo.core.neural_types import AudioSignal, LengthsType, NeuralType class AudioDataLayer(DataLayerNM): diff --git a/nemo/collections/tts/tacotron2_modules.py b/nemo/collections/tts/tacotron2_modules.py index 697c1aa3083b..dd0f56a18816 100644 --- a/nemo/collections/tts/tacotron2_modules.py +++ b/nemo/collections/tts/tacotron2_modules.py @@ -84,7 +84,7 @@ def input_ports(self): # ), # "embedding_length": NeuralType({0: AxisType(BatchTag)}), "char_phone_embeddings": NeuralType(EmbeddedTextType(), ('B', 'D', 'T')), - "embedding_length": NeuralType(LengthsType(), tuple('B')) + "embedding_length": NeuralType(LengthsType(), tuple('B')), } @property @@ -166,7 +166,7 @@ def input_ports(self): # ), "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), "encoded_length": NeuralType(LengthsType(), tuple('B')), - "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) + "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), } @property @@ -181,7 +181,7 @@ def output_ports(self): # "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), "gate_output": NeuralType(ChannelType(), ('B', 'T')), - "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')) + "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')), } def __init__( @@ -279,7 +279,7 @@ def input_ports(self): # ), # "encoded_length": NeuralType({0: AxisType(BatchTag)}), "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), - "encoded_length": NeuralType(LengthsType(), tuple('B')) + "encoded_length": NeuralType(LengthsType(), tuple('B')), } @property diff --git a/nemo/collections/tts/waveglow_modules.py b/nemo/collections/tts/waveglow_modules.py index 703b0e8f3458..06439d272ff2 100644 --- a/nemo/collections/tts/waveglow_modules.py +++ b/nemo/collections/tts/waveglow_modules.py @@ -150,7 +150,7 @@ def input_ports(self): def output_ports(self): """Returns definitions of module output ports. """ - #return {"audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} + # return {"audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} return {"audio": NeuralType(AudioSignal(), ('B', 'T'))} def __str__(self): diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 47bf93896dc8..017b8367d341 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -33,7 +33,7 @@ 'LogprobsType', 'LengthsType', 'EmbeddedTextType', - 'EncodedRepresentation' + 'EncodedRepresentation', ] import abc from abc import ABC, abstractmethod From 4800d9e5e2c0bac3190d0c273e31ca374007c4b8 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 10:34:51 -0800 Subject: [PATCH 13/70] chaning common collection Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 21 +- nemo/backends/pytorch/common/other.py | 307 +------------------------ nemo/backends/pytorch/common/rnn.py | 64 +----- nemo/backends/pytorch/common/search.py | 33 +-- 4 files changed, 31 insertions(+), 394 deletions(-) diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index 4cacb1853620..60b091802c68 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -105,19 +105,10 @@ class CrossEntropyLoss(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - logits: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - labels: - 0: AxisType(BatchTag) - """ return { - "logits": NeuralType(elements_type=LogitsType, axes=('B', 'D')), - "labels": NeuralType(elements_type=LabelsType, axes=tuple('B')), + "logits": NeuralType(elements_type=LogitsType(), axes=('B', 'D')), + "labels": NeuralType(elements_type=LabelsType(), axes=tuple('B')), } @property @@ -127,7 +118,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(elements_type=LossType)} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, weight=None): super().__init__() @@ -152,8 +143,8 @@ def input_ports(self): 0: AxisType(RegressionTag) """ return { - "preds": NeuralType(RegressionValuesType, tuple('B')), - "labels": NeuralType(LabelsType, tuple('B')), + "preds": NeuralType(RegressionValuesType(), tuple('B')), + "labels": NeuralType(LabelsType(), tuple('B')), } @property @@ -163,7 +154,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(elements_type=LossType)} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self): super().__init__() diff --git a/nemo/backends/pytorch/common/other.py b/nemo/backends/pytorch/common/other.py index 58790a1727be..7de337619f01 100644 --- a/nemo/backends/pytorch/common/other.py +++ b/nemo/backends/pytorch/common/other.py @@ -1,12 +1,7 @@ # Copyright (c) 2019 NVIDIA Corporation """Core PyTorch-base Neural Modules""" __all__ = [ - 'SimpleCombiner', - 'ArgMaxSimple', - 'TableLookUp', - 'TableLookUp2', 'SequenceEmbedding', - 'SequenceProjection', 'ZerosLikeNM', ] @@ -20,262 +15,20 @@ from nemo.core.neural_types import * -class SimpleCombiner(TrainableNM): - """Performs simple combination of two NmTensors. For example, it can - perform x1 + x2. - - Args: - mode (str): Can be ['add', 'sum', 'max']. - Defaults to 'add'. - - """ - - @property - def input_ports(self): - """Returns definitions of module input ports. - """ - if self._input_ports is None: - return {"x1": NeuralType(VoidType()), "x2": NeuralType(VoidType())} - else: - return self._input_ports - - @property - def output_ports(self): - """Returns definitions of module output ports. - - combined: - None - """ - if self._output_ports is None: - return {"combined": NeuralType(VoidType())} - else: - return self._output_ports - - def __init__(self, mode="add", input_ports=None, output_ports=None): - super().__init__() - self._mode = mode - self._input_ports = input_ports - self._output_ports = output_ports - - def forward(self, x1, x2): - if self._mode == "add" or self._mode == "sum": - return x1 + x2 - elif self._mode == "max": - return torch.max(x1, x2, out=None) - else: - raise NotImplementedError("SimpleCombiner does not have {0} mode".format(self._mode)) - - -class ArgMaxSimple(TrainableNM): # Notice TWO base classes - """ - """ - - @property - def input_ports(self): - """Returns definitions of module input ports. - - x: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - """ - return {"x": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} - - @property - def output_ports(self): - """Returns definitions of module output ports. - - values: - 0: AxisType(BatchTag) - - indices: - 0: AxisType(BatchTag) - """ - return { - "values": NeuralType({0: AxisType(BatchTag)}), - "indices": NeuralType({0: AxisType(BatchTag)}), - } - - def __init__(self): - super().__init__() - - # this method is key method you need to overwrite from PyTorch - # nn.Module's API - def forward(self, x): - values, indices = torch.max(x, 1) - return values, indices - - -class TableLookUp(NeuralModule): - """Performs a table lookup. For example, convert class ids to names""" - - def __init__(self, ids2classes=None): - NeuralModule.__init__(self) - - if ids2classes is None: - ids2classes = {} - self._ids2classes = ids2classes - - @property - def input_ports(self): - """Returns definitions of module input ports. - - indices: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - """ - return {"indices": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)})} - - @property - def output_ports(self): - """Returns definitions of module output ports. - - indices: - 0: AxisType(BatchTag) - 1: AxisType(TimeTag) - """ - return {"indices": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - - def set_weights(self, name2weight: Dict[(str, bool)], name2name_and_transform): - pass - - def tie_weights_with(self, module, weight_names): - pass - - def save_to(self, path): - pass - - def restore_from(self, path): - pass - - def freeze(self, weights: Set[str] = None): - pass - - def unfreeze(self, weights: Set[str] = None): - pass - - def __call__(self, force_pt=False, *input, **kwargs): - pt_call = len(input) > 0 or force_pt - if pt_call: - # [inds] = kwargs.values() - # np_inds = inds.detach().cpu().numpy().reshape(-1) - # result = [self._ids2classes[i] for i in np_inds] - # #result = list(map(lambda x: self._ids2classes[x], np_inds)) - # return result - inds = kwargs["indices"] - np_inds = inds.detach().transpose_(1, 0).cpu().numpy().tolist() - result = [] - for lst in np_inds: - sublst = [] - for tid in lst: - if tid != 1: - sublst.append(tid) - else: - break - result.append(list(map(lambda x: self._ids2classes[x], sublst))) - return [result] - else: - return NeuralModule.__call__(self, **kwargs) - - def parameters(self): - return None - - def get_weights(self) -> Iterable[Optional[Mapping]]: - return None - - -class TableLookUp2(NeuralModule): - """Performs a table lookup. For example, convert class ids to names""" - - def set_weights(self, name2weight: Dict[(str, bool)], name2name_and_transform): - pass - - def tie_weights_with(self, module, weight_names): - pass - - def save_to(self, path): - pass - - def restore_from(self, path): - pass - - def freeze(self, weights: Set[str] = None): - pass - - def unfreeze(self, weights: Set[str] = None): - pass - - @property - def input_ports(self): - """Returns definitions of module input ports. - - """ - return {} - - @property - def output_ports(self): - """Returns definitions of module output ports. - - classes: - None - """ - return {"classes": None} - - def __init__(self, detokenizer=None): - NeuralModule.__init__(self) - self._detokenizer = detokenizer - - def __call__(self, force_pt=False, *input, **kwargs): - pt_call = len(input) > 0 or force_pt - if pt_call: - # [inds] = kwargs.values() - inds = kwargs["indices"] - np_inds = inds.detach().cpu().numpy().tolist() - result = [] - for lst in np_inds: - sublst = [] - for tid in lst: - if tid != 1: - sublst.append(tid) - else: - break - result.append(self._detokenizer(sublst)) - return result - else: - return NeuralModule.__call__(self, **kwargs) - - def parameters(self): - return None - - def get_weights(self) -> Iterable[Optional[Mapping]]: - return None - - class SequenceEmbedding(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_seq: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) """ - return {"input_seq": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)})} + # return {"input_seq": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag)})} + return {"input_seq": NeuralModule(ChannelType(), ('T', 'B'))} @property def output_ports(self): """Returns definitions of module output ports. - - outputs: - 0: AxisType(TimeTag) - - 1: AxisType(BatchTag) - - 2: AxisType(ChannelTag) """ - return {"outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),})} + # return {"outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),})} + return {"outputs": NeuralType(ChannelType(), ('T', 'B', 'D'))} def __init__(self, voc_size, hidden_size, dropout=0.0): super().__init__() @@ -294,64 +47,20 @@ def forward(self, input_seq): return embedded -class SequenceProjection(TrainableNM): - @property - def input_ports(self): - """Returns definitions of module input ports. - - input_seq: - Empty Type?!? - """ - return {"input_seq": NeuralType({})} - - @property - def output_ports(self): - """Returns definitions of module output ports. - - outputs: - None - """ - return {"outputs": None} - - def __init__(self, from_dim, to_dim, dropout=0.0): - super().__init__() - - self.from_dim = from_dim - self.to_dim = to_dim - self.dropout = dropout - self.projection = nn.Linear(self.from_dim, self.to_dim, bias=False) - if self.dropout != 0.0: - self.embedding_dropout = nn.Dropout(self.dropout) - - def forward(self, input_seq): - p = self.projection(input_seq) - if self.dropout != 0.0: - p = self.dropout(p) - return p - - class ZerosLikeNM(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} + # return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} + return {"input_type_ids": NeuralType(VoidType(), ('B', 'T'))} @property def output_ports(self): """Returns definitions of module output ports. - - input_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ - return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} + # return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} + return {"input_type_ids": NeuralType(ChannelType(), ('B', 'T'))} def __init__(self): super().__init__() diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index 95724a9fa6ad..7777c699bb9a 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -67,18 +67,6 @@ class DecoderRNN(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - targets: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - encoder_outputs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ return { # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), @@ -92,20 +80,6 @@ def input_ports(self): @property def output_ports(self): """Returns definitions of module output ports. - - log_probs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - attention_weights: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(TimeTag) """ return { # 'log_probs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), @@ -231,45 +205,23 @@ class EncoderRNN(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - targets: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - encoder_outputs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ return { - 'inputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'input_lens': NeuralType({0: AxisType(BatchTag),}, optional=True), + # 'inputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'input_lens': NeuralType({0: AxisType(BatchTag),}, optional=True), + 'inputs': NeuralType(ChannelType(), ('B', 'T')), + 'input_lens': NeuralType(LengthsType(), tuple('B')), } @property def output_ports(self): """Returns definitions of module output ports. - - log_probs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) - - attention_weights: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(TimeTag) """ return { - 'outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - 'hidden': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # 'outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + # 'hidden': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), + 'outputs': NeuralType(ChannelType(), ('B', 'T', 'D')), + 'hidden': NeuralType(ChannelType(), ('B', 'T', 'D')), } def __init__( diff --git a/nemo/backends/pytorch/common/search.py b/nemo/backends/pytorch/common/search.py index f58cab7034d0..2051a648b6cb 100644 --- a/nemo/backends/pytorch/common/search.py +++ b/nemo/backends/pytorch/common/search.py @@ -3,7 +3,7 @@ import torch from nemo.backends.pytorch.nm import NonTrainableNM -from nemo.core import AxisType +from nemo.core.neural_types import ChannelType, NeuralType INF = float('inf') BIG_NUM = 1e4 @@ -31,39 +31,24 @@ class GreedySearch(NonTrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - encoder_outputs: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ return { - 'encoder_outputs': NeuralType( - {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, - ) + # 'encoder_outputs': NeuralType( + # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, + # ) + "encoder_outputs": NeuralType(ChannelType(), ('B', 'T', 'D'), optional=True) } @property def output_ports(self): """Returns definitions of module output ports. - predictions: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - attention_weights: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(TimeTag) """ return { - 'predictions': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'attention_weights': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), + # 'predictions': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # 'attention_weights': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), + "predictions": NeuralType(ChannelType(), ('B', 'T')), + "attention_weights": NeuralType(ChannelType(), ('B', 'T', 'T')), } def __init__(self, decoder, pad_id, bos_id, eos_id, max_len, batch_size=None): From 7b44c95ceab9a5ba7aa91084febb63d9eddeb46f Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 10:47:51 -0800 Subject: [PATCH 14/70] update gan collection Signed-off-by: Oleksii Kuchaiev --- nemo/collections/simple_gan/gan.py | 283 +++++++++++------------------ 1 file changed, 102 insertions(+), 181 deletions(-) diff --git a/nemo/collections/simple_gan/gan.py b/nemo/collections/simple_gan/gan.py index 16e83bbdf5c5..4ea0dc0bdb05 100644 --- a/nemo/collections/simple_gan/gan.py +++ b/nemo/collections/simple_gan/gan.py @@ -4,8 +4,9 @@ from torch.utils.data import Dataset from torchvision import datasets, transforms -from nemo.backends.pytorch.nm import DataLayerNM, LossNM, NonTrainableNM, TrainableNM -from nemo.core import AxisType, BatchTag, ChannelTag, DeviceType, HeightTag, NeuralType, WidthTag +from nemo.backends.pytorch.nm import DataLayerNM, LossNM, TrainableNM +from nemo.core import DeviceType +from nemo.core.neural_types import ChannelType, LabelsType, LossType, NeuralType class SimpleDiscriminator(TrainableNM): @@ -16,37 +17,25 @@ class SimpleDiscriminator(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - image: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) """ return { - "image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ) + # "image": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ) + "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) } @property def output_ports(self): """Returns definitions of module output ports. - - decision: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag, 1) """ - return {"decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)})} + # return {"decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)})} + return {"decision": NeuralType(ChannelType(), ('B', 'C'))} def __init__(self): super().__init__() @@ -78,49 +67,33 @@ class SimpleGenerator(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - latents: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag, 64) - - 2: AxisType(HeightTag, 4) - - 3: AxisType(WidthTag, 4) """ return { - "latents": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag, 64), - 2: AxisType(HeightTag, 4), - 3: AxisType(WidthTag, 4), - } - ) + # "latents": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag, 64), + # 2: AxisType(HeightTag, 4), + # 3: AxisType(WidthTag, 4), + # } + # ) + "latents": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) } @property def output_ports(self): """Returns definitions of module output ports. - - image: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) """ return { - "image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ) + # "image": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ) + "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) } def __init__(self): @@ -162,7 +135,8 @@ def input_ports(self): 1: AxisType(ChannelTag, 1) """ return { - "decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), + # "decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), + "decision": NeuralType(ChannelType(), ('B', 'D')) } @property @@ -197,31 +171,19 @@ class GradientPenalty(LossNM): @property def input_ports(self): """Returns definitions of module input ports. - - interpolated_image: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) - - interpolated_decision: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag, 1) """ return { - "interpolated_image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ), - "interpolated_decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), + # "interpolated_image": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ), + # "interpolated_decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), + "interpolated_image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), + "interpolated_decision": NeuralType(ChannelType(), ('B', 'C')), } @property @@ -231,7 +193,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, lambda_): super().__init__() @@ -266,66 +228,42 @@ class InterpolateImage(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - image1: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) - - image2: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) """ return { - "image1": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ), - "image2": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ), + # "image1": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ), + # "image2": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ), + "image1": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), + "image2": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), } @property def output_ports(self): """Returns definitions of module output ports. - - interpolated_image: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, 28) - - 3: AxisType(WidthTag, 28) """ return { - "interpolated_image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 28), - 3: AxisType(WidthTag, 28), - } - ) + # "interpolated_image": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, 28), + # 3: AxisType(WidthTag, 28), + # } + # ) + "interpolated_image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) } def __init__(self): @@ -362,14 +300,15 @@ def output_ports(self): 3: AxisType(WidthTag, 4) """ return { - "latent": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag, 64), - 2: AxisType(HeightTag, 4), - 3: AxisType(WidthTag, 4), - } - ) + # "latent": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag, 64), + # 2: AxisType(HeightTag, 4), + # 3: AxisType(WidthTag, 4), + # } + # ) + "latent": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) } def __init__(self, batch_size): @@ -415,46 +354,28 @@ class MnistGanDataLayer(DataLayerNM): @property def output_ports(self): """Returns definitions of module output ports. - - latent: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag, 64) - - 2: AxisType(HeightTag, 4) - - 3: AxisType(WidthTag, 4) - - image: - 0: AxisType(BatchTag) - - 1: AxisType(ChannelTag) - - 2: AxisType(HeightTag, user defined) - - 3: AxisType(WidthTag, user defined) - - label: - 0: AxisType(BatchTag) """ return { - "latent": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag, 64), - 2: AxisType(HeightTag, 4), - 3: AxisType(WidthTag, 4), - } - ), - "image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, self._input_size[1]), - 3: AxisType(WidthTag, self._input_size[0]), - } - ), - "label": NeuralType({0: AxisType(BatchTag)}), + # "latent": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag, 64), + # 2: AxisType(HeightTag, 4), + # 3: AxisType(WidthTag, 4), + # } + # ), + # "image": NeuralType( + # { + # 0: AxisType(BatchTag), + # 1: AxisType(ChannelTag), + # 2: AxisType(HeightTag, self._input_size[1]), + # 3: AxisType(WidthTag, self._input_size[0]), + # } + # ), + # "label": NeuralType({0: AxisType(BatchTag)}), + "latent": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), + "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), + "label": NeuralType(LabelsType(), tuple('B')), } def __init__(self, batch_size, root, train=True, shuffle=True): From 45f30ec2f8a3e16e2964c72dac13621aac8412a8 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 11:00:11 -0800 Subject: [PATCH 15/70] neural types fix in dialog Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/data_layers/state_tracking_trade_datalayer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index da51068b8519..28b43173711a 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -43,7 +43,7 @@ import nemo from nemo.collections.nlp.data.datasets import MultiWOZDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core.neural_types import ChannelType, LabelsType, NeuralType +from nemo.core.neural_types import ChannelType, LabelsType, NeuralType, LengthsType __all__ = ['MultiWOZDataLayer'] @@ -75,9 +75,9 @@ def output_ports(self): # "gating_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "turn_domain": NeuralType(None), "src_ids": NeuralType(ChannelType(), ('B', 'T')), - "src_lens": NeuralType(ChannelType(), tuple('B')), + "src_lens": NeuralType(LengthsType(), tuple('B')), "tgt_ids": NeuralType(ChannelType(), ('B', 'D', 'T')), - "tgt_lens": NeuralType(ChannelType(), ('B', 'D')), + "tgt_lens": NeuralType(LengthsType(), ('B', 'D')), "gating_labels": NeuralType(LabelsType(), ('B', 'D')), "turn_domain": NeuralType(), } From 396c427dba6ca8914f3924abb5a345df8ca3f2ac Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 11:01:04 -0800 Subject: [PATCH 16/70] fix types in dialog Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/data_layers/state_tracking_trade_datalayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index 28b43173711a..2cf2eb08951f 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -43,7 +43,7 @@ import nemo from nemo.collections.nlp.data.datasets import MultiWOZDataset from nemo.collections.nlp.nm.data_layers.text_datalayer import TextDataLayer -from nemo.core.neural_types import ChannelType, LabelsType, NeuralType, LengthsType +from nemo.core.neural_types import ChannelType, LabelsType, LengthsType, NeuralType __all__ = ['MultiWOZDataLayer'] From 6eca99481ca4aaf656312ff69f78982528d09076 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 13:47:53 -0800 Subject: [PATCH 17/70] fixing types in trade example Signed-off-by: Oleksii Kuchaiev --- nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py | 4 ++-- .../dialogue_state_tracking/state_tracking_trade_nm.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py index 662de183a183..015486be08e5 100644 --- a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py +++ b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py @@ -39,7 +39,7 @@ import torch from nemo.backends.pytorch.nm import LossNM -from nemo.core.neural_types import ChannelType, LabelsType, LogitsType, LossType, NeuralType +from nemo.core.neural_types import ChannelType, LabelsType, LengthsType, LogitsType, LossType, NeuralType __all__ = ['TRADEMaskedCrossEntropy', 'CrossEntropyLoss3D'] @@ -75,7 +75,7 @@ def input_ports(self): # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), "logits": NeuralType(LogitsType(), ('B', 'T', 'D', 'D')), "targets": NeuralType(ChannelType(), ('B', 'D', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'D')), + "loss_mask": NeuralType(LengthsType(), ('B', 'D')), } @property diff --git a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py index 0ac416d4ab73..fc5977c727b1 100644 --- a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py +++ b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py @@ -45,7 +45,7 @@ from torch import nn as nn from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import ChannelType, LengthsType, NeuralType +from nemo.core.neural_types import ChannelType, LengthsType, LogitsType, NeuralType __all__ = ['TRADEGenerator'] @@ -95,8 +95,8 @@ def output_ports(self): # 'gate_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), # } return { - 'point_outputs': NeuralType(ChannelType(), ('B', 'T', 'D', 'D')), - 'gate_outputs': NeuralType(ChannelType(), ('B', 'D', 'D')), + 'point_outputs': NeuralType(LogitsType(), ('B', 'T', 'D', 'D')), + 'gate_outputs': NeuralType(LogitsType(), ('B', 'D', 'D')), } def __init__(self, vocab, embeddings, hid_size, dropout, slots, nb_gate, teacher_forcing=0.5): From 9d202108d30e5ed52a4ae5ca9bd11969de8db99f Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 14:06:55 -0800 Subject: [PATCH 18/70] fix types in gleu scripts unittests in parallel Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 21 ++------------------- nemo/core/neural_types/elements.py | 2 +- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 61be16bc01de..d088e924bd38 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,6 @@ pipeline { disableConcurrentBuilds() } stages { - stage('PyTorch version') { steps { sh 'python -c "import torch; print(torch.__version__)"' @@ -24,27 +23,11 @@ pipeline { sh 'python setup.py style' } } - stage('Unittests Core') { - steps { - sh './reinstall.sh && python -m unittest tests/core/*.py' - } - } - stage('Unittests ASR') { + stage('Unittests') { steps { - sh 'python -m unittest tests/asr/*.py' + sh './reinstall.sh && python -m unittest' } } - stage('Unittests NLP') { - steps { - sh 'python -m unittest tests/nlp/*.py' - } - } - stage('Unittests TTS') { - steps { - sh 'python -m unittest tests/tts/*.py' - } - } - stage('Parallel Stage1') { failFast true parallel { diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 017b8367d341..ad66f5e7b654 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -176,7 +176,7 @@ def __str__(self): return "mfcc spectorgram type" -class PredictionsType(ElementType): +class PredictionsType(LabelsType): def __str__(self): return "predictions values type" From a6d779accf63c9e94ed2e9cc748996abe50625ea Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Fri, 7 Feb 2020 14:35:15 -0800 Subject: [PATCH 19/70] fix styles in asr postprocessing Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/losses/padded_smoothed_cross_entropy_loss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py index b56717af885d..1c14dbf545e2 100644 --- a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py +++ b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py @@ -17,7 +17,7 @@ from nemo.backends.pytorch import LossNM from nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss import SmoothedCrossEntropyLoss from nemo.collections.nlp.utils.common_nlp_utils import mask_padded_tokens -from nemo.core import AxisType, ChannelType, LogitsType, LossType, NeuralType +from nemo.core import LabelsType, LogitsType, LossType, NeuralType __all__ = ['PaddedSmoothedCrossEntropyLossNM'] @@ -43,7 +43,7 @@ def input_ports(self): # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "target_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "target_ids": NeuralType(ChannelType(), ('B', 'T')), + "target_ids": NeuralType(LabelsType(), ('B', 'T')), } @property From e5f0544475a5ef599e77ac7f86a66d7902c2c221 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Sat, 8 Feb 2020 13:59:10 -0800 Subject: [PATCH 20/70] fix types in intent classification Signed-off-by: Oleksii Kuchaiev --- .../joint_intent_slot_with_bert.py | 2 +- .../nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/nlp/intent_detection_slot_tagging/joint_intent_slot_with_bert.py b/examples/nlp/intent_detection_slot_tagging/joint_intent_slot_with_bert.py index 98a060f24ea0..0c0cc8689d3f 100644 --- a/examples/nlp/intent_detection_slot_tagging/joint_intent_slot_with_bert.py +++ b/examples/nlp/intent_detection_slot_tagging/joint_intent_slot_with_bert.py @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright 2019 AI Applications Design Team at NVIDIA. All Rights Reserved. +# Copyright 2019 NVIDIA. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py index faa273919d15..0f31bdab513c 100644 --- a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py +++ b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py @@ -63,7 +63,7 @@ def output_ports(self): # "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), "intent_logits": NeuralType(LogitsType(), ('B', 'D')), - "slot_logits": NeuralType(LogitsType(), ('B', 'D')), + "slot_logits": NeuralType(LogitsType(), ('B', 'T', 'D')), } def __init__(self, hidden_size, num_intents, num_slots, dropout=0.0, use_transformer_pretrained=True, **kwargs): From 3bfb9f96cb120f59e54a043d3aa2975f593a3bc9 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Sat, 8 Feb 2020 15:01:24 -0800 Subject: [PATCH 21/70] fix beam search Signed-off-by: Oleksii Kuchaiev --- nemo/collections/asr/beam_search_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/asr/beam_search_decoder.py b/nemo/collections/asr/beam_search_decoder.py index 70f0517330cd..793e899016f4 100644 --- a/nemo/collections/asr/beam_search_decoder.py +++ b/nemo/collections/asr/beam_search_decoder.py @@ -56,7 +56,7 @@ def output_ports(self): predictions: NeuralType(None) """ - return {"predictions": NeuralType(None)} + return {"predictions": NeuralType(VoidType())} def __init__(self, vocab, beam_width, alpha, beta, lm_path, num_cpus, cutoff_prob=1.0, cutoff_top_n=40): From 00445a75a07d0ef3ef52850d7fd4382670f3ff22 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Sat, 8 Feb 2020 15:22:39 -0800 Subject: [PATCH 22/70] los fix Signed-off-by: Oleksii Kuchaiev --- nemo/collections/simple_gan/gan.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nemo/collections/simple_gan/gan.py b/nemo/collections/simple_gan/gan.py index 4ea0dc0bdb05..4d8f48e6cdbb 100644 --- a/nemo/collections/simple_gan/gan.py +++ b/nemo/collections/simple_gan/gan.py @@ -142,11 +142,8 @@ def input_ports(self): @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(None)} + return {"loss": NeuralType(LossType())} def __init__(self, neg=False): super().__init__() From 2695e3354a996c17a0a91e95d3a53d4461795827 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 14:54:24 -0800 Subject: [PATCH 23/70] addressing comments from @yzhang123 and @blisc Signed-off-by: Oleksii Kuchaiev --- CHANGELOG.md | 4 ++++ nemo/backends/pytorch/tutorials/toys.py | 10 ++-------- nemo/collections/asr/audio_preprocessing.py | 9 --------- nemo/collections/asr/beam_search_decoder.py | 3 ++- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fbf6d6ac532..ce02933de720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,12 +70,16 @@ To release a new version, please update the changelog as followed: ## [Unreleased] ### Added +- New Neural Type System and its tests. +([PR #307](https://github.com/NVIDIA/NeMo/pull/307)) - @okuchaiev - Named tensors tuple module's output for graph construction. ([PR #268](https://github.com/NVIDIA/NeMo/pull/268)) - @stasbel - Introduced the `deprecated` decorator. ([PR #298](https://github.com/NVIDIA/NeMo/pull/298)) - @tkornuta-nvidia ### Changed +- All collections changed to use New Neural Type System. +([PR #307](https://github.com/NVIDIA/NeMo/pull/307)) - @okuchaiev - Additional Collections Repositories merged into core `nemo_toolkit` package. ([PR #289](https://github.com/NVIDIA/NeMo/pull/289)) - @DEKHTIARJonathan - Refactor manifest files parsing and processing for re-using. diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 324d26e50077..0708ea65beb9 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -189,11 +189,8 @@ def input_ports(self): @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(axes=None)} + return {"loss": NeuralType(LossType())} def __init__(self): super().__init__() @@ -216,11 +213,8 @@ def input_ports(self): @property def output_ports(self): """Returns definitions of module output ports. - - loss: - NeuralType(None) """ - return {"loss": NeuralType(axes=None)} + return {"loss": NeuralType(LossType())} def __init__(self): super().__init__() diff --git a/nemo/collections/asr/audio_preprocessing.py b/nemo/collections/asr/audio_preprocessing.py index 54ab17515b46..f2950162a346 100644 --- a/nemo/collections/asr/audio_preprocessing.py +++ b/nemo/collections/asr/audio_preprocessing.py @@ -121,15 +121,6 @@ class AudioToSpectrogramPreprocessor(AudioPreprocessor): @property def input_ports(self): """Returns definitions of module input ports. - - input_signal: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - length: - 0: AxisType(BatchTag) - """ return { # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), diff --git a/nemo/collections/asr/beam_search_decoder.py b/nemo/collections/asr/beam_search_decoder.py index 793e899016f4..2cb919ee4fe4 100644 --- a/nemo/collections/asr/beam_search_decoder.py +++ b/nemo/collections/asr/beam_search_decoder.py @@ -56,7 +56,8 @@ def output_ports(self): predictions: NeuralType(None) """ - return {"predictions": NeuralType(VoidType())} + # return {"predictions": NeuralType(VoidType())} + return {"predictions": NeuralType(PredictionsType(), ('B', 'T'))} def __init__(self, vocab, beam_width, alpha, beta, lm_path, num_cpus, cutoff_prob=1.0, cutoff_top_n=40): From 27f049e91c4c7b147158f2e69d1c8ec6aeb32f69 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 4 Feb 2020 09:45:33 -0800 Subject: [PATCH 24/70] init Signed-off-by: Yang Zhang --- .../question_answering_squad.py | 43 ++++++------- .../nlp/data/datasets/qa_squad_dataset.py | 9 ++- .../nlp/data/tokenizers/bert_tokenizer.py | 63 ++++++++++++++++++- nemo/collections/nlp/huggingface/__init__.py | 2 + 4 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 nemo/collections/nlp/huggingface/__init__.py diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 8b2104805932..44a01dd5df60 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -90,17 +90,11 @@ def parse_args(): ) parser.add_argument("--bert_config", default=None, type=str, help="Path to bert config file in json format") parser.add_argument( - "--tokenizer_model", - default="tokenizer.model", + "--model_type", + default="bert", type=str, - help="Path to pretrained tokenizer model," "only used if --tokenizer is sentencepiece", - ) - parser.add_argument( - "--tokenizer", - default="nemobert", - type=str, - choices=["nemobert", "sentencepiece"], - help="tokenizer to use, " "only relevant when using custom " "pretrained checkpoint.", + help="model type", + choices=['bert', 'roberta'] ) parser.add_argument("--optimizer_kind", default="adam", type=str, help="Optimizer kind") parser.add_argument("--lr_policy", default="WarmupAnnealing", type=str) @@ -244,7 +238,10 @@ def create_pipeline( [loss_output.start_logits, loss_output.end_logits, input_data.unique_ids], data_layer, ) - +MODEL_CLASSES = { + "bert": (nemo_nlp.NemoBertTokenizer), + "roberta": (nemo_nlp.NemoRobertaTokenizer), +} if __name__ == "__main__": args = parse_args() @@ -268,19 +265,17 @@ def create_pipeline( add_time_to_log_dir=False, ) - if args.tokenizer == "sentencepiece": - try: - tokenizer = nemo_nlp.data.utilsSentencePieceTokenizer(model_path=args.tokenizer_model) - except Exception: - raise ValueError( - "Using --tokenizer=sentencepiece \ - requires valid --tokenizer_model" - ) - tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) - elif args.tokenizer == "nemobert": - tokenizer = nemo_nlp.data.NemoBertTokenizer(args.pretrained_bert_model) - else: - raise ValueError(f"received unexpected tokenizer '{args.tokenizer}'") + # if args.tokenizer == "sentencepiece": + # try: + # tokenizer = nemo_nlp.SentencePieceTokenizer(model_path=args.tokenizer_model) + # except Exception: + # raise ValueError( + # "Using --tokenizer=sentencepiece \ + # requires valid --tokenizer_model" + # ) + # tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) + # else: + tokenizer = MODEL_CLASSES[args.model_type] if args.bert_config is not None: with open(args.bert_config) as json_file: diff --git a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py index 18109384b099..9014f2457c61 100644 --- a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py +++ b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py @@ -403,7 +403,14 @@ def evaluate( return exact_match, f1, all_predictions -def convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, has_groundtruth): +def convert_examples_to_features( + examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + has_groundtruth, +): """Loads a data file into a list of `InputBatch`s.""" unique_id = 1000000000 diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index c3c4b358030d..238499380fe2 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -16,7 +16,7 @@ import re -from transformers import BertTokenizer +from transformers import BertTokenizer, RobertaTokenizer from nemo.collections.nlp.data.tokenizers.tokenizer_spec import TokenizerSpec @@ -116,3 +116,64 @@ def bos_id(self): def eos_id(self): return self.tokens_to_ids(["[SEP]"])[0] + + + + +class NemoRobertaTokenizer(TokenizerSpec): + def __init__( + self, + pretrained_model, + do_lower_case=True, + never_split=("", "", "", "", "", ""), + ): + + self.tokenizer = RobertaTokenizer.from_pretrained(pretrained_model) + if "uncased" not in pretrained_model: + self.tokenizer.basic_tokenizer.do_lower_case = False + self.never_split = never_split + + def text_to_tokens(self, text): + tokens = self.tokenizer.tokenize(text) + return tokens + + def tokens_to_text(self, tokens): + text = self.tokenizer.convert_tokens_to_string(tokens) + return remove_spaces(handle_quotes(text.strip())) + + def token_to_id(self, token): + return self.tokens_to_ids([token])[0] + + def tokens_to_ids(self, tokens): + ids = self.tokenizer.convert_tokens_to_ids(tokens) + return ids + + def ids_to_tokens(self, ids): + tokens = self.tokenizer.convert_ids_to_tokens(ids) + return tokens + + def text_to_ids(self, text): + tokens = self.text_to_tokens(text) + ids = self.tokens_to_ids(tokens) + return ids + + def ids_to_text(self, ids): + tokens = self.ids_to_tokens(ids) + tokens_clean = [t for t in tokens if t not in self.never_split] + text = self.tokens_to_text(tokens_clean) + return text + + def pad_id(self): + return self.tokens_to_ids([""])[0] + + def bos_id(self): + return self.tokens_to_ids([""])[0] + + def eos_id(self): + return self.tokens_to_ids([""])[0] + + def sep_id(self): + return self.tokens_to_ids([""])[0] + + def cls_id(self): + return self.tokens_to_ids([""])[0] diff --git a/nemo/collections/nlp/huggingface/__init__.py b/nemo/collections/nlp/huggingface/__init__.py new file mode 100644 index 000000000000..5db46872cfe6 --- /dev/null +++ b/nemo/collections/nlp/huggingface/__init__.py @@ -0,0 +1,2 @@ +from .bert import BERT +from .roberta import RoBERTa From d4ace540dc1cf7655167fc83ff4028babe05f815 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 5 Feb 2020 17:44:17 -0800 Subject: [PATCH 25/70] added roberta and albert Signed-off-by: Yang Zhang --- .../question_answering_squad.py | 65 ++++++- .../nlp/data/datasets/qa_squad_dataset.py | 11 +- .../nlp/data/tokenizers/bert_tokenizer.py | 108 ++++------- .../nlp/nm/data_layers/qa_squad_datalayer.py | 2 +- .../trainables/common/huggingface/__init__.py | 2 + .../common/huggingface/albert_nm.py | 182 ++++++++++++++++++ .../trainables/common/huggingface/bert_nm.py | 2 +- .../common/huggingface/roberta_nm.py | 182 ++++++++++++++++++ 8 files changed, 467 insertions(+), 87 deletions(-) create mode 100644 nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py create mode 100644 nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 44a01dd5df60..ccd886af41ce 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -82,7 +82,7 @@ def parse_args(): "--dev_file", type=str, required=True, help="The evaluation data file. Should be *.json", ) parser.add_argument( - "--pretrained_bert_model", default="bert-base-uncased", type=str, help="Name of the pre-trained model" + "--pretrained_bert_model", type=str, help="Name of the pre-trained model" ) parser.add_argument("--checkpoint_dir", default=None, type=str, help="Checkpoint directory for inference.") parser.add_argument( @@ -94,7 +94,7 @@ def parse_args(): default="bert", type=str, help="model type", - choices=['bert', 'roberta'] + choices=['bert', 'roberta', 'albert'] ) parser.add_argument("--optimizer_kind", default="adam", type=str, help="Optimizer kind") parser.add_argument("--lr_policy", default="WarmupAnnealing", type=str) @@ -239,8 +239,49 @@ def create_pipeline( data_layer, ) MODEL_CLASSES = { - "bert": (nemo_nlp.NemoBertTokenizer), - "roberta": (nemo_nlp.NemoRobertaTokenizer), + "bert": { + "model_name": "bert-base-uncased", + "tokenizer_name": "bert-base-uncased", + "model": nemo_nlp.nm.trainables.huggingface.BERT, + "special_tokens": + {"unk_token": "[UNK]", + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token":"[MASK]", + "eos_token":"[SEP]", + "cls_token":"[CLS]", + }, + }, + "roberta": { + "model_name": "roberta-base", + "tokenizer_name": "roberta-base", + "model": nemo_nlp.nm.trainables.huggingface.Roberta, + "special_tokens": + {"unk_token": "", + "sep_token": "", + "pad_token": "", + "bos_token": "", + "mask_token":"", + "eos_token":"", + "cls_token":"", + }, + }, + "albert": { + "model_name": "albert-base-v2", + "tokenizer_name": "albert-base-v2", + "model": nemo_nlp.nm.trainables.huggingface.Albert, + "special_tokens": + { + "unk_token": "", + "sep_token": "[SEP]", + "eos_token": "[SEP]", + "pad_token": "", + "cls_token": "[CLS]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + }, + } } if __name__ == "__main__": @@ -275,18 +316,28 @@ def create_pipeline( # ) # tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) # else: - tokenizer = MODEL_CLASSES[args.model_type] + tokenizer_cls = nemo_nlp.data.NemoBertTokenizer + tokenizer_special_tokens = MODEL_CLASSES[args.model_type]["special_tokens"] + model_cls = MODEL_CLASSES[args.model_type]["model"] + model_name = MODEL_CLASSES[args.model_type]["model_name"] + tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] + + if args.pretrained_bert_model is None: + args.pretrained_bert_model = model_name + + tokenizer = tokenizer_cls(do_lower_case=args.do_lower_case, pretrained_model=tokenizer_name, + special_tokens=tokenizer_special_tokens, bert_derivate=args.model_type) if args.bert_config is not None: with open(args.bert_config) as json_file: config = json.load(json_file) - model = nemo_nlp.nm.trainables.huggingface.BERT(**config) + model = model_cls(**config) else: """ Use this if you're using a standard BERT model. To see the list of pretrained models, call: nemo_nlp.huggingface.BERT.list_pretrained_models() """ - model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name=args.pretrained_bert_model) + model = model_cls(pretrained_model_name=args.pretrained_bert_model) hidden_size = model.hidden_size diff --git a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py index 9014f2457c61..b2010b3654bf 100644 --- a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py +++ b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py @@ -474,12 +474,12 @@ def convert_examples_to_features( token_to_orig_map = {} token_is_max_context = {} segment_ids = [] - tokens.append("[CLS]") + tokens.append(tokenizer.bos_token) segment_ids.append(0) for token in query_tokens: tokens.append(token) segment_ids.append(0) - tokens.append("[SEP]") + tokens.append(tokenizer.sep_token) segment_ids.append(0) for i in range(doc_span.length): @@ -490,7 +490,7 @@ def convert_examples_to_features( token_is_max_context[len(tokens)] = is_max_context tokens.append(all_doc_tokens[split_token_index]) segment_ids.append(1) - tokens.append("[SEP]") + tokens.append(tokenizer.eos_token) segment_ids.append(1) input_ids = tokenizer.tokens_to_ids(tokens) @@ -501,7 +501,7 @@ def convert_examples_to_features( # Zero-pad up to the sequence length. while len(input_ids) < max_seq_length: - input_ids.append(0) + input_ids.append(tokenizer.pad_id) input_mask.append(0) segment_ids.append(0) @@ -616,6 +616,9 @@ class SquadProcessor(DataProcessor): Processor for the SQuAD data set. used by the version 1.1 and version 2.0 of SQuAD, respectively. """ + def __init__(self, data_file, mode): + self.data_file = data_file + self.mode = mode def __init__(self, data_file, mode): self.data_file = data_file diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index 238499380fe2..afbbbe8fad44 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -16,11 +16,11 @@ import re -from transformers import BertTokenizer, RobertaTokenizer +from transformers import BertTokenizer, RobertaTokenizer, AlbertTokenizer from nemo.collections.nlp.data.tokenizers.tokenizer_spec import TokenizerSpec -__all__ = ['NemoBertTokenizer'] +__all__ = ['NemoBertTokenizer',] def handle_quotes(text): @@ -60,79 +60,34 @@ def remove_spaces(text): class NemoBertTokenizer(TokenizerSpec): - def __init__( - self, - pretrained_model=None, - vocab_file=None, - do_lower_case=True, - max_len=None, - do_basic_tokenize=True, - never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"), - ): - if pretrained_model: - self.tokenizer = BertTokenizer.from_pretrained(pretrained_model) - if "uncased" not in pretrained_model: - self.tokenizer.basic_tokenizer.do_lower_case = False - else: - self.tokenizer = BertTokenizer(vocab_file, do_lower_case, do_basic_tokenize) - self.vocab_size = len(self.tokenizer.vocab) - self.never_split = never_split - - def text_to_tokens(self, text): - tokens = self.tokenizer.tokenize(text) - return tokens - - def tokens_to_text(self, tokens): - text = self.tokenizer.convert_tokens_to_string(tokens) - return remove_spaces(handle_quotes(text.strip())) - - def token_to_id(self, token): - return self.tokens_to_ids([token])[0] - - def tokens_to_ids(self, tokens): - ids = self.tokenizer.convert_tokens_to_ids(tokens) - return ids - - def ids_to_tokens(self, ids): - tokens = self.tokenizer.convert_ids_to_tokens(ids) - return tokens - - def text_to_ids(self, text): - tokens = self.text_to_tokens(text) - ids = self.tokens_to_ids(tokens) - return ids - - def ids_to_text(self, ids): - tokens = self.ids_to_tokens(ids) - tokens_clean = [t for t in tokens if t not in self.never_split] - text = self.tokens_to_text(tokens_clean) - return text - - def pad_id(self): - return self.tokens_to_ids(["[PAD]"])[0] - - def bos_id(self): - return self.tokens_to_ids(["[CLS]"])[0] - - def eos_id(self): - return self.tokens_to_ids(["[SEP]"])[0] - - - - -class NemoRobertaTokenizer(TokenizerSpec): def __init__( self, pretrained_model, + bert_derivate='bert', + special_tokens={ + "unk_token": "[UNK]", + "sep_token": "[SEP]", + "eos_token": "[SEP]", + "pad_token": "[PAD]", + "cls_token": "[CLS]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + }, do_lower_case=True, - never_split=("", "", "", "", "", ""), ): - self.tokenizer = RobertaTokenizer.from_pretrained(pretrained_model) - if "uncased" not in pretrained_model: - self.tokenizer.basic_tokenizer.do_lower_case = False - self.never_split = never_split - + if bert_derivate == 'bert': + tokenizer_cls = BertTokenizer + elif bert_derivate == 'albert': + tokenizer_cls = AlbertTokenizer + elif bert_derivate == 'roberta': + tokenizer_cls = RobertaTokenizer + + self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) + for k, v in special_tokens.items(): + setattr(self, k, v) + + self.never_split = tuple(special_tokens.values()) def text_to_tokens(self, text): tokens = self.tokenizer.tokenize(text) return tokens @@ -163,17 +118,22 @@ def ids_to_text(self, ids): text = self.tokens_to_text(tokens_clean) return text + @property def pad_id(self): - return self.tokens_to_ids([""])[0] + return self.tokens_to_ids([getattr(self, 'pad_token')])[0] + @property def bos_id(self): - return self.tokens_to_ids([""])[0] + return self.tokens_to_ids([getattr(self, 'bos_token')])[0] + @property def eos_id(self): - return self.tokens_to_ids([""])[0] + return self.tokens_to_ids([getattr(self, 'eos_token')])[0] + @property def sep_id(self): - return self.tokens_to_ids([""])[0] + return self.tokens_to_ids([getattr(self, 'sep_token')])[0] + @property def cls_id(self): - return self.tokens_to_ids([""])[0] + return self.tokens_to_ids([getattr(self, 'cls_token')])[0] diff --git a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py index 544d3da0f0ca..3d18df1a81c6 100644 --- a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py @@ -26,7 +26,7 @@ class BertQuestionAnsweringDataLayer(TextDataLayer): Creates the data layer to use for Question Answering classification task. Args: - data_file (str): data file. + data_file (str): Directory that contains train.*.json and dev.*.json. tokenizer (obj): Tokenizer object, e.g. NemoBertTokenizer. version_2_with_negative (bool): True if training should allow unanswerable questions. diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py b/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py index 847a4aba526f..a478bf48542c 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py @@ -15,3 +15,5 @@ # ============================================================================= from nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm import * +from nemo.collections.nlp.nm.trainables.common.huggingface.roberta_nm import * +from nemo.collections.nlp.nm.trainables.common.huggingface.albert_nm import * diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py new file mode 100644 index 000000000000..bd220cca1beb --- /dev/null +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py @@ -0,0 +1,182 @@ +# ============================================================================= +# Copyright 2019 AI Applications Design Team at NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from typing import List, Optional + +from transformers import ( + ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertConfig, + AlbertModel, +) + +from nemo.backends.pytorch.nm import TrainableNM +from nemo.core.neural_modules import PretrainedModelInfo +from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag + +__all__ = ['Albert'] + + +class Albert(TrainableNM): + """ + ALBERT wraps around the Huggingface implementation of ALBERT from their + transformers repository for easy use within NeMo. + + Args: + pretrained_model_name (str): If using a pretrained model, this should + be the model's name. Otherwise, should be left as None. + config_filename (str): path to model configuration file. Optional. + vocab_size (int): Size of the vocabulary file, if not using a + pretrained model. + hidden_size (int): Size of the encoder and pooler layers. + num_hidden_layers (int): Number of hidden layers in the encoder. + num_attention_heads (int): Number of attention heads for each layer. + intermediate_size (int): Size of intermediate layers in the encoder. + hidden_act (str): Activation function for encoder and pooler layers; + "gelu", "relu", and "swish" are supported. + max_position_embeddings (int): The maximum number of tokens in a + sequence. + """ + + @property + def input_ports(self): + """Returns definitions of module input ports. + + input_ids: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + token_type_ids: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + attention_mask: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + """ + return { + "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + } + + @property + def output_ports(self): + """Returns definitions of module output ports. + + hidden_states: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + 2: AxisType(ChannelTag) + """ + return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + + def __init__( + self, + pretrained_model_name=None, + config_filename=None, + vocab_size=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + max_position_embeddings=512, + ): + super().__init__() + + # Check that only one of pretrained_model_name, config_filename, and + # vocab_size was passed in + total = 0 + if pretrained_model_name is not None: + total += 1 + if config_filename is not None: + total += 1 + if vocab_size is not None: + total += 1 + + if total != 1: + raise ValueError( + "Only one of pretrained_model_name, vocab_size, " + + "or config_filename should be passed into the " + + "ALBERT constructor." + ) + + # TK: The following code checks the same once again. + if vocab_size is not None: + config = AlbertConfig( + vocab_size_or_config_json_file=vocab_size, + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + max_position_embeddings=max_position_embeddings, + ) + model = AlbertModel(config) + elif pretrained_model_name is not None: + model = AlbertModel.from_pretrained(pretrained_model_name) + elif config_filename is not None: + config = AlbertConfig.from_json_file(config_filename) + model = AlbertModel(config) + else: + raise ValueError( + "Either pretrained_model_name or vocab_size must" + " be passed into the ALBERT constructor" + ) + + model.to(self._device) + + self.add_module("albert", model) + self.config = model.config + + # TK: storing config name in init_params instead. + # for key, value in self.config.to_dict().items(): + # self._local_parameters[key] = value + + # Store the only value that will be used externally - hidden_size. + self._hidden_size = model.config.hidden_size + + @property + def hidden_size(self): + """ + Property returning hidden size. + + Returns: + Hidden size. + """ + return self._hidden_size + + @staticmethod + def list_pretrained_models() -> Optional[List[PretrainedModelInfo]]: + pretrained_models = [] + for key, value in ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.items(): + model_info = PretrainedModelInfo( + pretrained_model_name=key, + description="weights by HuggingFace", + parameters=ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP[key], + location=value, + ) + pretrained_models.append(model_info) + return pretrained_models + + def forward(self, input_ids, token_type_ids, attention_mask): + return self.albert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)[0] \ No newline at end of file diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py index ba41297e13b1..cd9ffb162973 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py @@ -148,7 +148,7 @@ def __init__( # self._local_parameters[key] = value # Store the only value that will be used externally - hidden_size. - self._hidden_size = hidden_size + self._hidden_size = model.config.hidden_size @property def hidden_size(self): diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py new file mode 100644 index 000000000000..00e91f6735ec --- /dev/null +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py @@ -0,0 +1,182 @@ +# ============================================================================= +# Copyright 2019 AI Applications Design Team at NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +from typing import List, Optional + +from transformers import ( + ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + RobertaConfig, + RobertaModel, +) + +from nemo.backends.pytorch.nm import TrainableNM +from nemo.core.neural_modules import PretrainedModelInfo +from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag + +__all__ = ['Roberta'] + + +class Roberta(TrainableNM): + """ + ROBERTA wraps around the Huggingface implementation of ROBERTA from their + transformers repository for easy use within NeMo. + + Args: + pretrained_model_name (str): If using a pretrained model, this should + be the model's name. Otherwise, should be left as None. + config_filename (str): path to model configuration file. Optional. + vocab_size (int): Size of the vocabulary file, if not using a + pretrained model. + hidden_size (int): Size of the encoder and pooler layers. + num_hidden_layers (int): Number of hidden layers in the encoder. + num_attention_heads (int): Number of attention heads for each layer. + intermediate_size (int): Size of intermediate layers in the encoder. + hidden_act (str): Activation function for encoder and pooler layers; + "gelu", "relu", and "swish" are supported. + max_position_embeddings (int): The maximum number of tokens in a + sequence. + """ + + @property + def input_ports(self): + """Returns definitions of module input ports. + + input_ids: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + token_type_ids: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + attention_mask: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + """ + return { + "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + } + + @property + def output_ports(self): + """Returns definitions of module output ports. + + hidden_states: + 0: AxisType(BatchTag) + + 1: AxisType(TimeTag) + + 2: AxisType(ChannelTag) + """ + return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + + def __init__( + self, + pretrained_model_name=None, + config_filename=None, + vocab_size=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + max_position_embeddings=512, + ): + super().__init__() + + # Check that only one of pretrained_model_name, config_filename, and + # vocab_size was passed in + total = 0 + if pretrained_model_name is not None: + total += 1 + if config_filename is not None: + total += 1 + if vocab_size is not None: + total += 1 + + if total != 1: + raise ValueError( + "Only one of pretrained_model_name, vocab_size, " + + "or config_filename should be passed into the " + + "ROBERTA constructor." + ) + + # TK: The following code checks the same once again. + if vocab_size is not None: + config = RobertaConfig( + vocab_size_or_config_json_file=vocab_size, + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + max_position_embeddings=max_position_embeddings, + ) + model = RobertaModel(config) + elif pretrained_model_name is not None: + model = RobertaModel.from_pretrained(pretrained_model_name) + elif config_filename is not None: + config = RobertaConfig.from_json_file(config_filename) + model = RobertaModel(config) + else: + raise ValueError( + "Either pretrained_model_name or vocab_size must" + " be passed into the ROBERTA constructor" + ) + + model.to(self._device) + + self.add_module("roberta", model) + self.config = model.config + + # TK: storing config name in init_params instead. + # for key, value in self.config.to_dict().items(): + # self._local_parameters[key] = value + + # Store the only value that will be used externally - hidden_size. + self._hidden_size = model.config.hidden_size + + @property + def hidden_size(self): + """ + Property returning hidden size. + + Returns: + Hidden size. + """ + return self._hidden_size + + @staticmethod + def list_pretrained_models() -> Optional[List[PretrainedModelInfo]]: + pretrained_models = [] + for key, value in ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.items(): + model_info = PretrainedModelInfo( + pretrained_model_name=key, + description="weights by HuggingFace", + parameters=ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP[key], + location=value, + ) + pretrained_models.append(model_info) + return pretrained_models + + def forward(self, input_ids, token_type_ids, attention_mask): + return self.roberta(input_ids, attention_mask=attention_mask)[0] \ No newline at end of file From 50d4482af8d64a42de6417784c53306553668ce6 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 4 Feb 2020 09:45:33 -0800 Subject: [PATCH 26/70] init Signed-off-by: Yang Zhang --- examples/nlp/question_answering/question_answering_squad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index ccd886af41ce..68ac577be44e 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -445,6 +445,7 @@ def create_pipeline( null_score_diff_threshold=args.null_score_diff_threshold, do_lower_case=args.do_lower_case, ) + logging.info(f"exact_match: {exact_match}, f1: {f1}") if args.output_prediction_file is not None: with open(args.output_prediction_file, "w") as writer: From c67f3812980db7085f3d14119b5b5fd0bbd2ff54 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 5 Feb 2020 17:44:17 -0800 Subject: [PATCH 27/70] added roberta and albert Signed-off-by: Yang Zhang --- examples/nlp/question_answering/question_answering_squad.py | 6 +++++- nemo/collections/nlp/nm/losses/qa_squad_loss.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 68ac577be44e..1cf5a902b2c1 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -24,8 +24,12 @@ To finetune Squad v1.1 on pretrained BERT large uncased on 1 GPU: python question_answering_squad.py +<<<<<<< 50d4482af8d64a42de6417784c53306553668ce6 --train_file /path_to_data_dir/squad/v1.1/train-v1.1.json --dev_file /path_to_data_dir/squad/v1.1/dev-v1.1.json +======= +--data_file /path_to_data_file/squad/v1.1 +>>>>>>> added roberta and albert --work_dir /path_to_output_folder --bert_checkpoint /path_to_bert_checkpoint --amp_opt_level "O1" @@ -445,7 +449,7 @@ def create_pipeline( null_score_diff_threshold=args.null_score_diff_threshold, do_lower_case=args.do_lower_case, ) - + logging.info(f"exact_match: {exact_match}, f1: {f1}") if args.output_prediction_file is not None: with open(args.output_prediction_file, "w") as writer: diff --git a/nemo/collections/nlp/nm/losses/qa_squad_loss.py b/nemo/collections/nlp/nm/losses/qa_squad_loss.py index 0919d7362e90..053130bdf255 100644 --- a/nemo/collections/nlp/nm/losses/qa_squad_loss.py +++ b/nemo/collections/nlp/nm/losses/qa_squad_loss.py @@ -96,6 +96,7 @@ def _loss_function(self, **kwargs): start_positions = start_positions.squeeze(-1) if len(end_positions.size()) > 1: end_positions = end_positions.squeeze(-1) + import ipdb; ipdb.set_trace() ignored_index = start_logits.size(1) start_positions.clamp_(0, ignored_index) end_positions.clamp_(0, ignored_index) From ea3223c49c66dd83cfd6d5285eb60560ded9b8d0 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 7 Feb 2020 13:31:27 -0800 Subject: [PATCH 28/70] adding roberta and albert nm Signed-off-by: Yang Zhang --- nemo/collections/nlp/nm/losses/qa_squad_loss.py | 1 - .../nlp/nm/trainables/common/huggingface/albert_nm.py | 2 +- .../nlp/nm/trainables/common/huggingface/roberta_nm.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nemo/collections/nlp/nm/losses/qa_squad_loss.py b/nemo/collections/nlp/nm/losses/qa_squad_loss.py index 053130bdf255..0919d7362e90 100644 --- a/nemo/collections/nlp/nm/losses/qa_squad_loss.py +++ b/nemo/collections/nlp/nm/losses/qa_squad_loss.py @@ -96,7 +96,6 @@ def _loss_function(self, **kwargs): start_positions = start_positions.squeeze(-1) if len(end_positions.size()) > 1: end_positions = end_positions.squeeze(-1) - import ipdb; ipdb.set_trace() ignored_index = start_logits.size(1) start_positions.clamp_(0, ignored_index) end_positions.clamp_(0, ignored_index) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py index bd220cca1beb..9252e47ae2be 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py @@ -179,4 +179,4 @@ def list_pretrained_models() -> Optional[List[PretrainedModelInfo]]: return pretrained_models def forward(self, input_ids, token_type_ids, attention_mask): - return self.albert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)[0] \ No newline at end of file + return self.albert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)[0] diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py index 00e91f6735ec..eed3a559f09c 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py @@ -179,4 +179,4 @@ def list_pretrained_models() -> Optional[List[PretrainedModelInfo]]: return pretrained_models def forward(self, input_ids, token_type_ids, attention_mask): - return self.roberta(input_ids, attention_mask=attention_mask)[0] \ No newline at end of file + return self.roberta(input_ids, attention_mask=attention_mask)[0] From b6c2fef4a1aa04633fdd695843647f6fd35523fd Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 7 Feb 2020 15:43:18 -0800 Subject: [PATCH 29/70] remove file Signed-off-by: Yang Zhang --- nemo/collections/nlp/huggingface/__init__.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 nemo/collections/nlp/huggingface/__init__.py diff --git a/nemo/collections/nlp/huggingface/__init__.py b/nemo/collections/nlp/huggingface/__init__.py deleted file mode 100644 index 5db46872cfe6..000000000000 --- a/nemo/collections/nlp/huggingface/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .bert import BERT -from .roberta import RoBERTa From cba60548993fdbe3f17fb30bb6a95ae9d01d959c Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 13:19:41 -0800 Subject: [PATCH 30/70] change data saving path Signed-off-by: Yang Zhang --- examples/nlp/scripts/get_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/nlp/scripts/get_squad.py b/examples/nlp/scripts/get_squad.py index 85534f14848d..cc31f20eea1d 100755 --- a/examples/nlp/scripts/get_squad.py +++ b/examples/nlp/scripts/get_squad.py @@ -62,7 +62,7 @@ def download(self): type=str, required=False, help='directory to store data', - default=os.path.split(os.path.abspath(__file__))[0] + '../../../../../../examples/data/lm', + default=os.path.split(os.path.abspath(__file__))[0], ) args = parser.parse_args() logging.info(args.destDir) From ab7e755bdb84d90453d6aa306f44d44cfb1c07c1 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 17:29:54 -0800 Subject: [PATCH 31/70] rebase master Signed-off-by: Yang Zhang --- .../question_answering_squad.py | 81 +++++++++---------- .../nlp/data/datasets/qa_squad_dataset.py | 8 +- .../nlp/data/tokenizers/bert_tokenizer.py | 13 +-- .../trainables/common/huggingface/__init__.py | 2 +- 4 files changed, 51 insertions(+), 53 deletions(-) diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 1cf5a902b2c1..f4592654e9b6 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -85,20 +85,14 @@ def parse_args(): parser.add_argument( "--dev_file", type=str, required=True, help="The evaluation data file. Should be *.json", ) - parser.add_argument( - "--pretrained_bert_model", type=str, help="Name of the pre-trained model" - ) + parser.add_argument("--pretrained_bert_model", type=str, help="Name of the pre-trained model") parser.add_argument("--checkpoint_dir", default=None, type=str, help="Checkpoint directory for inference.") parser.add_argument( "--bert_checkpoint", default=None, type=str, help="Path to BERT model checkpoint for finetuning." ) parser.add_argument("--bert_config", default=None, type=str, help="Path to bert config file in json format") parser.add_argument( - "--model_type", - default="bert", - type=str, - help="model type", - choices=['bert', 'roberta', 'albert'] + "--model_type", default="bert", type=str, help="model type", choices=['bert', 'roberta', 'albert'] ) parser.add_argument("--optimizer_kind", default="adam", type=str, help="Optimizer kind") parser.add_argument("--lr_policy", default="WarmupAnnealing", type=str) @@ -242,41 +236,42 @@ def create_pipeline( [loss_output.start_logits, loss_output.end_logits, input_data.unique_ids], data_layer, ) + + MODEL_CLASSES = { "bert": { - "model_name": "bert-base-uncased", - "tokenizer_name": "bert-base-uncased", - "model": nemo_nlp.nm.trainables.huggingface.BERT, - "special_tokens": - {"unk_token": "[UNK]", - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token":"[MASK]", - "eos_token":"[SEP]", - "cls_token":"[CLS]", - }, + "model_name": "bert-base-uncased", + "tokenizer_name": "bert-base-uncased", + "model": nemo_nlp.nm.trainables.huggingface.BERT, + "special_tokens": { + "unk_token": "[UNK]", + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + }, }, "roberta": { - "model_name": "roberta-base", - "tokenizer_name": "roberta-base", - "model": nemo_nlp.nm.trainables.huggingface.Roberta, - "special_tokens": - {"unk_token": "", - "sep_token": "", - "pad_token": "", - "bos_token": "", - "mask_token":"", - "eos_token":"", - "cls_token":"", - }, + "model_name": "roberta-base", + "tokenizer_name": "roberta-base", + "model": nemo_nlp.nm.trainables.huggingface.Roberta, + "special_tokens": { + "unk_token": "", + "sep_token": "", + "pad_token": "", + "bos_token": "", + "mask_token": "", + "eos_token": "", + "cls_token": "", + }, }, "albert": { - "model_name": "albert-base-v2", - "tokenizer_name": "albert-base-v2", - "model": nemo_nlp.nm.trainables.huggingface.Albert, - "special_tokens": - { + "model_name": "albert-base-v2", + "tokenizer_name": "albert-base-v2", + "model": nemo_nlp.nm.trainables.huggingface.Albert, + "special_tokens": { "unk_token": "", "sep_token": "[SEP]", "eos_token": "[SEP]", @@ -284,8 +279,8 @@ def create_pipeline( "cls_token": "[CLS]", "bos_token": "[CLS]", "mask_token": "[MASK]", - }, - } + }, + }, } if __name__ == "__main__": @@ -329,8 +324,12 @@ def create_pipeline( if args.pretrained_bert_model is None: args.pretrained_bert_model = model_name - tokenizer = tokenizer_cls(do_lower_case=args.do_lower_case, pretrained_model=tokenizer_name, - special_tokens=tokenizer_special_tokens, bert_derivate=args.model_type) + tokenizer = tokenizer_cls( + do_lower_case=args.do_lower_case, + pretrained_model=tokenizer_name, + special_tokens=tokenizer_special_tokens, + bert_derivate=args.model_type, + ) if args.bert_config is not None: with open(args.bert_config) as json_file: diff --git a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py index b2010b3654bf..86e2a9f4060b 100644 --- a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py +++ b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py @@ -404,12 +404,7 @@ def evaluate( def convert_examples_to_features( - examples, - tokenizer, - max_seq_length, - doc_stride, - max_query_length, - has_groundtruth, + examples, tokenizer, max_seq_length, doc_stride, max_query_length, has_groundtruth, ): """Loads a data file into a list of `InputBatch`s.""" @@ -616,6 +611,7 @@ class SquadProcessor(DataProcessor): Processor for the SQuAD data set. used by the version 1.1 and version 2.0 of SQuAD, respectively. """ + def __init__(self, data_file, mode): self.data_file = data_file self.mode = mode diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index afbbbe8fad44..6b256ba2fa67 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -16,11 +16,13 @@ import re -from transformers import BertTokenizer, RobertaTokenizer, AlbertTokenizer +from transformers import AlbertTokenizer, BertTokenizer, RobertaTokenizer from nemo.collections.nlp.data.tokenizers.tokenizer_spec import TokenizerSpec -__all__ = ['NemoBertTokenizer',] +__all__ = [ + 'NemoBertTokenizer', +] def handle_quotes(text): @@ -72,7 +74,7 @@ def __init__( "cls_token": "[CLS]", "bos_token": "[CLS]", "mask_token": "[MASK]", - }, + }, do_lower_case=True, ): @@ -83,11 +85,12 @@ def __init__( elif bert_derivate == 'roberta': tokenizer_cls = RobertaTokenizer - self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) + self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) for k, v in special_tokens.items(): setattr(self, k, v) - + self.never_split = tuple(special_tokens.values()) + def text_to_tokens(self, text): tokens = self.tokenizer.tokenize(text) return tokens diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py b/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py index a478bf48542c..d71ca17ce84b 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/__init__.py @@ -14,6 +14,6 @@ # limitations under the License. # ============================================================================= +from nemo.collections.nlp.nm.trainables.common.huggingface.albert_nm import * from nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm import * from nemo.collections.nlp.nm.trainables.common.huggingface.roberta_nm import * -from nemo.collections.nlp.nm.trainables.common.huggingface.albert_nm import * From e963bb48e0f0e70736db0a77d95b53d8ae3cdfbe Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 19:25:48 -0800 Subject: [PATCH 32/70] wip pr Signed-off-by: Yang Zhang --- .../glue_benchmark_with_bert.py | 10 ++- .../nlp/language_modeling/bert_pretraining.py | 15 +++- .../question_answering_squad.py | 54 ++++++++---- .../punctuation_capitalization.py | 11 ++- .../token_classification.py | 9 +- .../nlp/data/datasets/datasets_utils.py | 4 +- .../nlp/data/datasets/lm_bert_dataset.py | 15 ++-- .../nlp/data/tokenizers/bert_tokenizer.py | 1 + .../tokenizers/sentencepiece_tokenizer.py | 87 ++++++++++++------- tests/nlp/test_spc_tokenizer.py | 72 +++++++++++---- 10 files changed, 195 insertions(+), 83 deletions(-) diff --git a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py index a7d909d93247..a221853427b3 100644 --- a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py +++ b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py @@ -199,8 +199,14 @@ Replace BERT-STEP-150000.pt with the path to your checkpoint. """ if args.tokenizer == "sentencepiece": - tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model) - tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) else: diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 1b6945801813..ab16f9a4287c 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -160,14 +160,21 @@ args.max_seq_length = config['max_position_embeddings'] if not args.preprocessed_data: - special_tokens = ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]'] + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } data_desc = BERTPretrainingDataDesc( - args.dataset_name, args.data_dir, args.vocab_size, args.sample_size, special_tokens, 'train.txt' + args.dataset_name, args.data_dir, args.vocab_size, args.sample_size, list(set(special_tokens.values())), 'train.txt' ) if args.tokenizer == "sentence-piece": logging.info("To use SentencePieceTokenizer.") - tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=data_desc.tokenizer_model) - tokenizer.add_special_tokens(special_tokens) + tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=data_desc.tokenizer_model, special_tokens=special_tokens) + # import ipdb; ipdb.set_trace() elif args.tokenizer == "nemo-bert": logging.info("To use NemoBertTokenizer.") vocab_file = os.path.join(args.data_dir, 'vocab.txt') diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index f4592654e9b6..8d221e614bee 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -24,12 +24,8 @@ To finetune Squad v1.1 on pretrained BERT large uncased on 1 GPU: python question_answering_squad.py -<<<<<<< 50d4482af8d64a42de6417784c53306553668ce6 --train_file /path_to_data_dir/squad/v1.1/train-v1.1.json --dev_file /path_to_data_dir/squad/v1.1/dev-v1.1.json -======= ---data_file /path_to_data_file/squad/v1.1 ->>>>>>> added roberta and albert --work_dir /path_to_output_folder --bert_checkpoint /path_to_bert_checkpoint --amp_opt_level "O1" @@ -94,6 +90,19 @@ def parse_args(): parser.add_argument( "--model_type", default="bert", type=str, help="model type", choices=['bert', 'roberta', 'albert'] ) + parser.add_argument( + "--tokenizer_model", + default="tokenizer.model", + type=str, + help="Path to pretrained tokenizer model, only used if --tokenizer is sentencepiece", +) + parser.add_argument( + "--tokenizer", + default="nemobert", + type=str, + choices=["nemobert", "sentencepiece"], + help="tokenizer to use, only relevant when using custom pretrained checkpoint.", + ) parser.add_argument("--optimizer_kind", default="adam", type=str, help="Optimizer kind") parser.add_argument("--lr_policy", default="WarmupAnnealing", type=str) parser.add_argument("--lr", default=3e-5, type=float, help="The initial learning rate.") @@ -305,21 +314,28 @@ def create_pipeline( add_time_to_log_dir=False, ) - # if args.tokenizer == "sentencepiece": - # try: - # tokenizer = nemo_nlp.SentencePieceTokenizer(model_path=args.tokenizer_model) - # except Exception: - # raise ValueError( - # "Using --tokenizer=sentencepiece \ - # requires valid --tokenizer_model" - # ) - # tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) - # else: - tokenizer_cls = nemo_nlp.data.NemoBertTokenizer - tokenizer_special_tokens = MODEL_CLASSES[args.model_type]["special_tokens"] - model_cls = MODEL_CLASSES[args.model_type]["model"] - model_name = MODEL_CLASSES[args.model_type]["model_name"] - tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] + if args.tokenizer == "sentencepiece": + try: + tokenizer = nemo_nlp.SentencePieceTokenizer(model_path=args.tokenizer_model) + except Exception: + raise ValueError( + "Using --tokenizer=sentencepiece \ + requires valid --tokenizer_model" + ) + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer.add_special_tokens(special_tokens) + else: + tokenizer_cls = nemo_nlp.data.NemoBertTokenizer + tokenizer_special_tokens = MODEL_CLASSES[args.model_type]["special_tokens"] + model_cls = MODEL_CLASSES[args.model_type]["model"] + model_name = MODEL_CLASSES[args.model_type]["model_name"] + tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] if args.pretrained_bert_model is None: args.pretrained_bert_model = model_name diff --git a/examples/nlp/token_classification/punctuation_capitalization.py b/examples/nlp/token_classification/punctuation_capitalization.py index 9b613f07ba16..ba2afcbc49d2 100644 --- a/examples/nlp/token_classification/punctuation_capitalization.py +++ b/examples/nlp/token_classification/punctuation_capitalization.py @@ -134,8 +134,15 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model) - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) else: diff --git a/examples/nlp/token_classification/token_classification.py b/examples/nlp/token_classification/token_classification.py index 8c8b62dde67f..d5e75c2984af 100644 --- a/examples/nlp/token_classification/token_classification.py +++ b/examples/nlp/token_classification/token_classification.py @@ -121,8 +121,15 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model) - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) else: diff --git a/nemo/collections/nlp/data/datasets/datasets_utils.py b/nemo/collections/nlp/data/datasets/datasets_utils.py index 872c62a263ba..4d7bf571035a 100644 --- a/nemo/collections/nlp/data/datasets/datasets_utils.py +++ b/nemo/collections/nlp/data/datasets/datasets_utils.py @@ -944,10 +944,12 @@ def get_intent_labels(intent_file): def download_wkt2(data_dir): + if os.path.exists(data_dir): + return os.makedirs('data/lm', exist_ok=True) logging.warning(f'Data not found at {data_dir}. ' f'Downloading wikitext-2 to data/lm') data_dir = 'data/lm/wikitext-2' - subprocess.call('scripts/get_wkt2.sh') + subprocess.call('../scripts/get_wkt2.sh') return data_dir diff --git a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py index 070b6d8753a7..2acc25401a95 100644 --- a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py +++ b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py @@ -46,9 +46,6 @@ def __init__( sentence_idx_file=None, ): self.tokenizer = tokenizer - self.cls_id = tokenizer.token_to_id("[CLS]") - self.sep_id = tokenizer.token_to_id("[SEP]") - self.pad_id = tokenizer.token_to_id("[PAD]") # Loading enormous datasets into RAM isn't always feasible -- for # example, the pubmed corpus is 200+ GB, which doesn't fit into RAM on @@ -254,8 +251,8 @@ def truncate_seq_pair(a, b, max_num_tokens): trunc_document.pop() truncate_seq_pair(a_document, b_document, max_num_tokens) - - output_ids = [self.cls_id] + a_document + [self.sep_id] + b_document + [self.sep_id] + + output_ids = [self.tokenizer.cls_id] + a_document + [self.tokenizer.sep_id] + b_document + [self.tokenizer.eos_id] input_ids, output_mask = self.mask_ids(output_ids) @@ -267,8 +264,8 @@ def truncate_seq_pair(a, b, max_num_tokens): padding_length = max(0, self.max_seq_length - len(input_ids)) if padding_length > 0: - input_ids.extend([self.pad_id] * padding_length) - output_ids.extend([self.pad_id] * padding_length) + input_ids.extend([self.tokenizer.pad_id] * padding_length) + output_ids.extend([self.tokenizer.pad_id] * padding_length) output_mask.extend([0] * padding_length) # TODO: wrap the return value with () for consistent style. @@ -311,7 +308,7 @@ def mask_ids(self, ids): mask_id = self.tokenizer.token_to_id("[MASK]") for word_ids in cand_indexes: - is_special = (word_ids[0] == self.cls_id) or (word_ids[0] == self.sep_id) + is_special = (word_ids[0] == self.tokenizer.cls_id) or (word_ids[0] == self.tokenizer.sep_id) if is_special or (random.random() > self.mask_probability): output_mask.extend([0] * len(word_ids)) masked_ids.extend(word_ids) @@ -326,7 +323,7 @@ def mask_ids(self, ids): for _ in word_ids: # randomly select a valid word random_word = random.randrange(self.vocab_size) - while random_word in (self.cls_id, self.sep_id): + while random_word in (self.tokenizer.cls_id, self.tokenizer.sep_id): random_word = random.randrange(self.vocab_size) masked_ids.append(random_word) # for 10%, use same token diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index 6b256ba2fa67..c52692f9dcae 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -86,6 +86,7 @@ def __init__( tokenizer_cls = RobertaTokenizer self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) + self.vocab_size = len(self.tokenizer.vocab) for k, v in special_tokens.items(): setattr(self, k, v) diff --git a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py index 29844e46ffd8..ce87c0762c22 100644 --- a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py @@ -22,13 +22,16 @@ class SentencePieceTokenizer(TokenizerSpec): - def __init__(self, model_path): + def __init__(self, model_path, special_tokens={}): self.tokenizer = spm.SentencePieceProcessor() self.tokenizer.Load(model_path) + # wihtout special tokens self.original_vocab_size = self.tokenizer.get_piece_size() self.vocab_size = self.tokenizer.get_piece_size() - self.special_tokens = {} - self.special_token_ids = {} + self.special_token_to_id = {} + self.id_to_special_token = {} + self.add_special_tokens(special_tokens) + def text_to_tokens(self, text): tokens = [] @@ -38,7 +41,7 @@ def text_to_tokens(self, text): while 1: indices = {} - for token in self.special_tokens: + for token in self.special_token_to_id: try: indices[token] = text[idx:].index(token) except ValueError: @@ -57,9 +60,6 @@ def text_to_tokens(self, text): tokens.extend(self.tokenizer.encode_as_pieces(text[idx:])) return tokens - def tokens_to_text(self, tokens): - return self.tokenizer.decode_pieces(tokens) - def text_to_ids(self, text): ids = [] idx = 0 @@ -68,7 +68,7 @@ def text_to_ids(self, text): while 1: indices = {} - for token in self.special_tokens: + for token in self.special_token_to_id: try: indices[token] = text[idx:].index(token) except ValueError: @@ -81,56 +81,83 @@ def text_to_ids(self, text): next_idx = idx + indices[next_token] ids.extend(self.tokenizer.encode_as_ids(text[idx:next_idx])) - ids.append(self.special_tokens[next_token]) + ids.append(self.special_token_to_id[next_token]) idx = next_idx + len(next_token) ids.extend(self.tokenizer.encode_as_ids(text[idx:])) return ids + def tokens_to_text(self, tokens): + return self.tokenizer.decode_pieces(tokens) + + def ids_to_text(self, ids): text = "" last_i = 0 for i, id in enumerate(ids): - if id in self.special_token_ids: + if id in self.id_to_special_token: text += self.tokenizer.decode_ids(ids[last_i:i]) + " " - text += self.special_token_ids[id] + " " + text += self.id_to_special_token[id] + " " last_i = i + 1 text += self.tokenizer.decode_ids(ids[last_i:]) return text.strip() - def token_to_id(self, token): - if token in self.special_tokens: - return self.special_tokens[token] - - return self.tokenizer.piece_to_id(token) def tokens_to_ids(self, tokens): ids = [] - for token in tokens: - if token in self.special_tokens: - ids.append(self.special_tokens[token]) - else: - ids.append(self.tokenizer.piece_to_id(token)) - + ids.append(self.token_to_id(token)) return ids + def token_to_id(self, token): + if token in self.special_token_to_id: + return self.special_token_to_id[token] + return self.tokenizer.piece_to_id(token) + def ids_to_tokens(self, ids): tokens = [] - for id in ids: if id >= self.original_vocab_size: - tokens.append(self.special_token_ids[id]) + tokens.append(self.id_to_special_token[id]) else: tokens.append(self.tokenizer.id_to_piece(id)) - return tokens def add_special_tokens(self, special_tokens): - for token in special_tokens: - if self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id(): - self.special_tokens[token] = self.vocab_size - self.special_token_ids[self.vocab_size] = token - self.vocab_size += 1 + if isinstance(special_tokens, list): + for token in special_tokens: + if self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() and token not in self.special_token_to_id: + self.special_token_to_id[token] = self.vocab_size + self.id_to_special_token[self.vocab_size] = token + self.vocab_size += 1 + elif isinstance(special_tokens, dict): + for token_name, token in special_tokens.items(): + if self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() and token not in self.special_token_to_id: + setattr(self, token_name, token) + self.special_token_to_id[token] = self.vocab_size + self.id_to_special_token[self.vocab_size] = token + self.vocab_size += 1 + + + + @property + def pad_id(self): + return self.tokens_to_ids([getattr(self, 'pad_token')])[0] + + @property + def bos_id(self): + return self.tokens_to_ids([getattr(self, 'bos_token')])[0] + + @property + def eos_id(self): + return self.tokens_to_ids([getattr(self, 'eos_token')])[0] + + @property + def sep_id(self): + return self.tokens_to_ids([getattr(self, 'sep_token')])[0] + + @property + def cls_id(self): + return self.tokens_to_ids([getattr(self, 'cls_token')])[0] diff --git a/tests/nlp/test_spc_tokenizer.py b/tests/nlp/test_spc_tokenizer.py index 6ec6e3c1541c..912a09ebf83b 100644 --- a/tests/nlp/test_spc_tokenizer.py +++ b/tests/nlp/test_spc_tokenizer.py @@ -23,16 +23,28 @@ class TestSPCTokenizer(NeMoUnitTest): def test_add_special_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) - - self.assertTrue(tokenizer.vocab_size == tokenizer.original_vocab_size + len(special_tokens)) + self.assertTrue(tokenizer.vocab_size == tokenizer.original_vocab_size + len(set(special_tokens.values()))) def test_text_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -55,21 +67,36 @@ def test_tokens_to_text(self): def test_text_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" ids = tokenizer.text_to_ids(text) self.assertTrue(len(ids) == len(text.split())) - self.assertTrue(ids.count(tokenizer.special_tokens["[CLS]"]) == 1) - self.assertTrue(ids.count(tokenizer.special_tokens["[MASK]"]) == 1) - self.assertTrue(ids.count(tokenizer.special_tokens["[SEP]"]) == 2) + self.assertTrue(ids.count(tokenizer.token_to_id("[CLS]")) == 1) + self.assertTrue(ids.count(tokenizer.token_to_id("[MASK]")) == 1) + self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]"))== 2) def test_ids_to_text(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -81,7 +108,15 @@ def test_ids_to_text(self): def test_tokens_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -89,14 +124,21 @@ def test_tokens_to_ids(self): ids = tokenizer.tokens_to_ids(tokens) self.assertTrue(len(ids) == len(tokens)) - self.assertTrue(ids.count(tokenizer.special_tokens["[CLS]"]) == 1) - self.assertTrue(ids.count(tokenizer.special_tokens["[MASK]"]) == 1) - self.assertTrue(ids.count(tokenizer.special_tokens["[SEP]"]) == 2) + self.assertTrue(ids.count(tokenizer.token_to_id("[CLS]")) == 1) + self.assertTrue(ids.count(tokenizer.token_to_id("[MASK]")) == 1) + self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]"))== 2) def test_ids_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = ["[CLS]", "[MASK]", "[SEP]"] + special_tokens= { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" From 68bfc9f7537e27255f169b036ef63c26ecbe9ac6 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 20:23:27 -0800 Subject: [PATCH 33/70] fix style Signed-off-by: Yang Zhang --- .../glue_benchmark_with_bert.py | 2 +- .../nlp/language_modeling/bert_pretraining.py | 25 ++++++++++++------- .../question_answering_squad.py | 12 ++++----- .../punctuation_capitalization.py | 2 +- .../token_classification.py | 2 +- .../nlp/data/datasets/datasets_utils.py | 2 +- .../nlp/data/datasets/lm_bert_dataset.py | 6 +++-- .../tokenizers/sentencepiece_tokenizer.py | 15 +++++------ tests/nlp/test_spc_tokenizer.py | 18 ++++++------- 9 files changed, 46 insertions(+), 38 deletions(-) diff --git a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py index a221853427b3..a6aa20380b9d 100644 --- a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py +++ b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py @@ -199,7 +199,7 @@ Replace BERT-STEP-150000.pt with the path to your checkpoint. """ if args.tokenizer == "sentencepiece": - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index ab16f9a4287c..5a7880be70fa 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -160,20 +160,27 @@ args.max_seq_length = config['max_position_embeddings'] if not args.preprocessed_data: - special_tokens= { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", + special_tokens = { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", } data_desc = BERTPretrainingDataDesc( - args.dataset_name, args.data_dir, args.vocab_size, args.sample_size, list(set(special_tokens.values())), 'train.txt' + args.dataset_name, + args.data_dir, + args.vocab_size, + args.sample_size, + list(set(special_tokens.values())), + 'train.txt', ) if args.tokenizer == "sentence-piece": logging.info("To use SentencePieceTokenizer.") - tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=data_desc.tokenizer_model, special_tokens=special_tokens) + tokenizer = nemo_nlp.data.SentencePieceTokenizer( + model_path=data_desc.tokenizer_model, special_tokens=special_tokens + ) # import ipdb; ipdb.set_trace() elif args.tokenizer == "nemo-bert": logging.info("To use NemoBertTokenizer.") diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 8d221e614bee..ffe87f08abc9 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -91,11 +91,11 @@ def parse_args(): "--model_type", default="bert", type=str, help="model type", choices=['bert', 'roberta', 'albert'] ) parser.add_argument( - "--tokenizer_model", - default="tokenizer.model", - type=str, - help="Path to pretrained tokenizer model, only used if --tokenizer is sentencepiece", -) + "--tokenizer_model", + default="tokenizer.model", + type=str, + help="Path to pretrained tokenizer model, only used if --tokenizer is sentencepiece", + ) parser.add_argument( "--tokenizer", default="nemobert", @@ -322,7 +322,7 @@ def create_pipeline( "Using --tokenizer=sentencepiece \ requires valid --tokenizer_model" ) - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", diff --git a/examples/nlp/token_classification/punctuation_capitalization.py b/examples/nlp/token_classification/punctuation_capitalization.py index ba2afcbc49d2..755bf64c9600 100644 --- a/examples/nlp/token_classification/punctuation_capitalization.py +++ b/examples/nlp/token_classification/punctuation_capitalization.py @@ -134,7 +134,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", diff --git a/examples/nlp/token_classification/token_classification.py b/examples/nlp/token_classification/token_classification.py index d5e75c2984af..265e43efe7be 100644 --- a/examples/nlp/token_classification/token_classification.py +++ b/examples/nlp/token_classification/token_classification.py @@ -121,7 +121,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", diff --git a/nemo/collections/nlp/data/datasets/datasets_utils.py b/nemo/collections/nlp/data/datasets/datasets_utils.py index 4d7bf571035a..8f4e0640245d 100644 --- a/nemo/collections/nlp/data/datasets/datasets_utils.py +++ b/nemo/collections/nlp/data/datasets/datasets_utils.py @@ -945,7 +945,7 @@ def get_intent_labels(intent_file): def download_wkt2(data_dir): if os.path.exists(data_dir): - return + return os.makedirs('data/lm', exist_ok=True) logging.warning(f'Data not found at {data_dir}. ' f'Downloading wikitext-2 to data/lm') data_dir = 'data/lm/wikitext-2' diff --git a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py index 2acc25401a95..b542d08adcce 100644 --- a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py +++ b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py @@ -251,8 +251,10 @@ def truncate_seq_pair(a, b, max_num_tokens): trunc_document.pop() truncate_seq_pair(a_document, b_document, max_num_tokens) - - output_ids = [self.tokenizer.cls_id] + a_document + [self.tokenizer.sep_id] + b_document + [self.tokenizer.eos_id] + + output_ids = ( + [self.tokenizer.cls_id] + a_document + [self.tokenizer.sep_id] + b_document + [self.tokenizer.eos_id] + ) input_ids, output_mask = self.mask_ids(output_ids) diff --git a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py index ce87c0762c22..37f1f22970a7 100644 --- a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py @@ -32,7 +32,6 @@ def __init__(self, model_path, special_tokens={}): self.id_to_special_token = {} self.add_special_tokens(special_tokens) - def text_to_tokens(self, text): tokens = [] idx = 0 @@ -90,7 +89,6 @@ def text_to_ids(self, text): def tokens_to_text(self, tokens): return self.tokenizer.decode_pieces(tokens) - def ids_to_text(self, ids): text = "" last_i = 0 @@ -104,7 +102,6 @@ def ids_to_text(self, ids): text += self.tokenizer.decode_ids(ids[last_i:]) return text.strip() - def tokens_to_ids(self, tokens): ids = [] for token in tokens: @@ -128,20 +125,24 @@ def ids_to_tokens(self, ids): def add_special_tokens(self, special_tokens): if isinstance(special_tokens, list): for token in special_tokens: - if self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() and token not in self.special_token_to_id: + if ( + self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() + and token not in self.special_token_to_id + ): self.special_token_to_id[token] = self.vocab_size self.id_to_special_token[self.vocab_size] = token self.vocab_size += 1 elif isinstance(special_tokens, dict): for token_name, token in special_tokens.items(): - if self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() and token not in self.special_token_to_id: + if ( + self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() + and token not in self.special_token_to_id + ): setattr(self, token_name, token) self.special_token_to_id[token] = self.vocab_size self.id_to_special_token[self.vocab_size] = token self.vocab_size += 1 - - @property def pad_id(self): return self.tokens_to_ids([getattr(self, 'pad_token')])[0] diff --git a/tests/nlp/test_spc_tokenizer.py b/tests/nlp/test_spc_tokenizer.py index 912a09ebf83b..dcae02027eb0 100644 --- a/tests/nlp/test_spc_tokenizer.py +++ b/tests/nlp/test_spc_tokenizer.py @@ -23,7 +23,7 @@ class TestSPCTokenizer(NeMoUnitTest): def test_add_special_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", @@ -37,7 +37,7 @@ def test_add_special_tokens(self): def test_text_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", @@ -67,7 +67,7 @@ def test_tokens_to_text(self): def test_text_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", @@ -83,13 +83,12 @@ def test_text_to_ids(self): self.assertTrue(len(ids) == len(text.split())) self.assertTrue(ids.count(tokenizer.token_to_id("[CLS]")) == 1) self.assertTrue(ids.count(tokenizer.token_to_id("[MASK]")) == 1) - self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]"))== 2) + self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]")) == 2) def test_ids_to_text(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", @@ -108,8 +107,7 @@ def test_ids_to_text(self): def test_tokens_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", @@ -126,12 +124,12 @@ def test_tokens_to_ids(self): self.assertTrue(len(ids) == len(tokens)) self.assertTrue(ids.count(tokenizer.token_to_id("[CLS]")) == 1) self.assertTrue(ids.count(tokenizer.token_to_id("[MASK]")) == 1) - self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]"))== 2) + self.assertTrue(ids.count(tokenizer.token_to_id("[SEP]")) == 2) def test_ids_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens= { + special_tokens = { "sep_token": "[SEP]", "pad_token": "[PAD]", "bos_token": "[CLS]", From 8f88a195a83c3374b8ee55544ce53e098b1479b9 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 20:39:44 -0800 Subject: [PATCH 34/70] fix bug Signed-off-by: Yang Zhang --- nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py index 37f1f22970a7..8405682854e5 100644 --- a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py @@ -134,11 +134,11 @@ def add_special_tokens(self, special_tokens): self.vocab_size += 1 elif isinstance(special_tokens, dict): for token_name, token in special_tokens.items(): + setattr(self, token_name, token) if ( self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() and token not in self.special_token_to_id ): - setattr(self, token_name, token) self.special_token_to_id[token] = self.vocab_size self.id_to_special_token[self.vocab_size] = token self.vocab_size += 1 From 14c0254fe4fa4afcd675308cda093355df775f46 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 21:06:16 -0800 Subject: [PATCH 35/70] debug Signed-off-by: Yang Zhang --- Jenkinsfile | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3bd1946fe5fb..ca2aadc22022 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,9 +21,30 @@ pipeline { } stage('Code formatting checks') { steps { - sh 'python setup.py style' + sh 'python setup.py style && ./reinstall' } } + + stage('Parallel NLP-BERT pretraining') { + failFast true + parallel { + stage('BERT on the fly preprocessing') { + steps { + sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=0 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wikitext-2 --dataset_name wikitext-2 --work_dir outputs/bert_lm/wikitext2 --batch_size 64 --lr 0.01 --lr_policy CosineAnnealing --lr_warmup_proportion 0.05 --tokenizer sentence-piece --vocab_size 3200 --hidden_size 768 --intermediate_size 3072 --num_hidden_layers 6 --num_attention_heads 12 --hidden_act "gelu" --save_step_freq 200 --sample_size 10000000 --mask_probability 0.15 --short_seq_prob 0.1 --max_steps=300' + sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wikitext2/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 8.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' + sh 'rm -rf examples/nlp/language_modeling/outputs/wikitext2' + } + } + stage('BERT offline preprocessing') { + steps { + sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=1 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wiki_book_mini --work_dir outputs/bert_lm/wiki_book --batch_size 8 --config_file /home/mrjenkins/TestData/nlp/bert_configs/uncased_L-12_H-768_A-12.json --save_step_freq 200 --max_steps 300 --num_gpus 1 --batches_per_step 1 --lr_policy SquareRootAnnealing --beta2 0.999 --beta1 0.9 --lr_warmup_proportion 0.01 --optimizer adam_w --weight_decay 0.01 --lr 0.875e-4 --preprocessed_data ' + sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wiki_book/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 15.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' + sh 'rm -rf examples/nlp/language_modeling/outputs/wiki_book' + } + } + } + } + stage('Unittests general') { steps { sh './reinstall.sh && python -m unittest tests/*.py' @@ -62,25 +83,7 @@ pipeline { } } - stage('Parallel NLP-BERT pretraining') { - failFast true - parallel { - stage('BERT on the fly preprocessing') { - steps { - sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=0 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wikitext-2 --dataset_name wikitext-2 --work_dir outputs/bert_lm/wikitext2 --batch_size 64 --lr 0.01 --lr_policy CosineAnnealing --lr_warmup_proportion 0.05 --tokenizer sentence-piece --vocab_size 3200 --hidden_size 768 --intermediate_size 3072 --num_hidden_layers 6 --num_attention_heads 12 --hidden_act "gelu" --save_step_freq 200 --sample_size 10000000 --mask_probability 0.15 --short_seq_prob 0.1 --max_steps=300' - sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wikitext2/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 8.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' - sh 'rm -rf examples/nlp/language_modeling/outputs/wikitext2' - } - } - stage('BERT offline preprocessing') { - steps { - sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=1 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wiki_book_mini --work_dir outputs/bert_lm/wiki_book --batch_size 8 --config_file /home/mrjenkins/TestData/nlp/bert_configs/uncased_L-12_H-768_A-12.json --save_step_freq 200 --max_steps 300 --num_gpus 1 --batches_per_step 1 --lr_policy SquareRootAnnealing --beta2 0.999 --beta1 0.9 --lr_warmup_proportion 0.01 --optimizer adam_w --weight_decay 0.01 --lr 0.875e-4 --preprocessed_data ' - sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wiki_book/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 15.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' - sh 'rm -rf examples/nlp/language_modeling/outputs/wiki_book' - } - } - } - } + stage('Parallel NLP Examples 1') { failFast true From f77ce1d60b9016c9887bf60a21b0b52839d6a131 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 21:09:00 -0800 Subject: [PATCH 36/70] debug Signed-off-by: Yang Zhang --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca2aadc22022..086228d981f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,13 @@ pipeline { } stage('Code formatting checks') { steps { - sh 'python setup.py style && ./reinstall' + sh 'python setup.py style' + } + } + + stage('install') { + steps { + sh './reinstall' } } From 256159b91764050d21949f8f4061ea355e6cff33 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 21:09:59 -0800 Subject: [PATCH 37/70] debug Signed-off-by: Yang Zhang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 086228d981f3..6af429e2c2f7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ pipeline { stage('install') { steps { - sh './reinstall' + sh './reinstall.sh' } } From 27a920f3f38c70bb0e31177b8d9a9527659c3892 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 10 Feb 2020 21:16:03 -0800 Subject: [PATCH 38/70] debug Signed-off-by: Yang Zhang --- examples/nlp/language_modeling/bert_pretraining.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 5a7880be70fa..428d2e5cb1c7 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -178,6 +178,7 @@ ) if args.tokenizer == "sentence-piece": logging.info("To use SentencePieceTokenizer.") + print(nemo_nlp.__file__) tokenizer = nemo_nlp.data.SentencePieceTokenizer( model_path=data_desc.tokenizer_model, special_tokens=special_tokens ) From 781ebc5810c885522bc3f0a6b71b5cb842f84c04 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 13:07:27 -0800 Subject: [PATCH 39/70] address review's comments Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/tutorials/chatbot/modules.py | 2 +- nemo/backends/pytorch/tutorials/toys.py | 2 +- nemo/collections/asr/losses.py | 2 +- .../dialogue_state_tracking/state_tracking_trade_nm.py | 5 +++-- nemo/core/neural_types/__init__.py | 8 ++++---- nemo/core/neural_types/elements.py | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nemo/backends/pytorch/tutorials/chatbot/modules.py b/nemo/backends/pytorch/tutorials/chatbot/modules.py index d665fc840fb0..ca37d874ec52 100644 --- a/nemo/backends/pytorch/tutorials/chatbot/modules.py +++ b/nemo/backends/pytorch/tutorials/chatbot/modules.py @@ -76,7 +76,7 @@ def input_ports(self): """ return { "input_seq": NeuralType(ChannelType(), ('T', 'B')), - "input_lengths": NeuralType(ChannelType(), tuple('B')), + "input_lengths": NeuralType(LengthsType(), tuple('B')), } @property diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 0708ea65beb9..8b18d9890c60 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -241,7 +241,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(axes=None)} + return {"loss": NeuralType(LossType())} def __init__(self): # Neural Module API specific diff --git a/nemo/collections/asr/losses.py b/nemo/collections/asr/losses.py index c29a0dba78be..a18f7c14aac2 100644 --- a/nemo/collections/asr/losses.py +++ b/nemo/collections/asr/losses.py @@ -25,7 +25,7 @@ def input_ports(self): # "input_length": NeuralType({0: AxisType(BatchTag)}), # "target_length": NeuralType({0: AxisType(BatchTag)}), "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), - "targets": NeuralType(ChannelType(), ('B', 'T')), + "targets": NeuralType(PredictionsType(), ('B', 'T')), "input_length": NeuralType(LengthsType(), tuple('B')), "target_length": NeuralType(LengthsType(), tuple('B')), } diff --git a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py index da0a7f0682ce..bd29209918c2 100644 --- a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py +++ b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py @@ -45,7 +45,7 @@ from torch import nn as nn from nemo.backends.pytorch.nm import TrainableNM -from nemo.core.neural_types import ChannelType, LengthsType, LogitsType, NeuralType +from nemo.core.neural_types import ChannelType, LabelsType, LengthsType, LogitsType, NeuralType __all__ = ['TRADEGenerator'] @@ -76,7 +76,8 @@ def input_ports(self): 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'C')), 'input_lens': NeuralType(LengthsType(), tuple('B')), 'src_ids': NeuralType(ChannelType(), ('B', 'T')), - 'targets': NeuralType(ChannelType(), ('B', 'D', 'T')), + # 'targets': NeuralType(ChannelType(), ('B', 'D', 'T')), + 'targets': NeuralType(LabelsType(), ('B', 'D', 'T')), } @property diff --git a/nemo/core/neural_types/__init__.py b/nemo/core/neural_types/__init__.py index 124adc132c72..1fb5bf349076 100644 --- a/nemo/core/neural_types/__init__.py +++ b/nemo/core/neural_types/__init__.py @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .axes import * -from .comparison import * -from .elements import * -from .neural_type import * +from nemo.core.neural_types.axes import * +from nemo.core.neural_types.comparison import * +from nemo.core.neural_types.elements import * +from nemo.core.neural_types.neural_type import * diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index ad66f5e7b654..dd7fcd754f98 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -39,7 +39,7 @@ from abc import ABC, abstractmethod from typing import Dict, Optional, Tuple -from .comparison import NeuralTypeComparisonResult +from nemo.core.neural_types.comparison import NeuralTypeComparisonResult class ElementType(ABC): From 51120d90d0f2597d8c63db52855dc299c54e17aa Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 13:25:06 -0800 Subject: [PATCH 40/70] fix unittests Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/rnn.py | 2 +- nemo/collections/asr/data_layer.py | 2 +- nemo/collections/asr/losses.py | 2 +- tests/asr/test_zeroDS.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index 43348d3d3d2c..774e11807edf 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -70,7 +70,7 @@ def input_ports(self): """ return { # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'targets': NeuralType(ChannelType(), ('B', 'T')), + 'targets': NeuralType(LabelsType(), ('B', 'T')), # 'encoder_outputs': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, # ), diff --git a/nemo/collections/asr/data_layer.py b/nemo/collections/asr/data_layer.py index 20df98b2add7..83d959a09974 100644 --- a/nemo/collections/asr/data_layer.py +++ b/nemo/collections/asr/data_layer.py @@ -102,7 +102,7 @@ def output_ports(self): # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), 'audio_signal': NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), 'a_sig_length': NeuralType(LengthsType(), tuple('B')), - 'transcripts': NeuralType(ChannelType(), ('B', 'T')), + 'transcripts': NeuralType(LabelsType(), ('B', 'T')), 'transcript_length': NeuralType(LengthsType(), tuple('B')), } diff --git a/nemo/collections/asr/losses.py b/nemo/collections/asr/losses.py index a18f7c14aac2..a9b77fe03e0b 100644 --- a/nemo/collections/asr/losses.py +++ b/nemo/collections/asr/losses.py @@ -25,7 +25,7 @@ def input_ports(self): # "input_length": NeuralType({0: AxisType(BatchTag)}), # "target_length": NeuralType({0: AxisType(BatchTag)}), "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), - "targets": NeuralType(PredictionsType(), ('B', 'T')), + "targets": NeuralType(LabelsType(), ('B', 'T')), "input_length": NeuralType(LengthsType(), tuple('B')), "target_length": NeuralType(LengthsType(), tuple('B')), } diff --git a/tests/asr/test_zeroDS.py b/tests/asr/test_zeroDS.py index 7c45720fc18f..e2c9bd6f7373 100644 --- a/tests/asr/test_zeroDS.py +++ b/tests/asr/test_zeroDS.py @@ -112,7 +112,7 @@ def test_asr_with_zero_ds(self): (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 64), AxisType(AxisKind.Time, 64)), ), "processed_length": NeuralType(LengthsType(), tuple('B')), - "transcript": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64))), + "transcript": NeuralType(LabelsType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64))), "transcript_length": NeuralType(LengthsType(), tuple('B')), }, ) From 8a3b53af946768306108399af115a2b1e19dace0 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 14:43:31 -0800 Subject: [PATCH 41/70] fix syntax erros Signed-off-by: Yang Zhang --- examples/nlp/asr_postprocessor/asr_postprocessor.py | 8 ++++---- .../language_modeling/language_modeling_transformer.py | 2 +- .../machine_translation_tutorial.py | 10 +++++----- .../nlp/callbacks/machine_translation_callback.py | 2 +- .../nlp/data/datasets/lm_transformer_dataset.py | 4 ++-- .../nlp/data/datasets/machine_translation_dataset.py | 8 ++++---- nemo/collections/nlp/data/tokenizers/char_tokenizer.py | 3 +++ nemo/collections/nlp/data/tokenizers/gpt2_tokenizer.py | 3 +++ nemo/collections/nlp/data/tokenizers/word_tokenizer.py | 3 +++ .../nlp/data/tokenizers/youtokentome_tokenizer.py | 3 +++ 10 files changed, 29 insertions(+), 17 deletions(-) diff --git a/examples/nlp/asr_postprocessor/asr_postprocessor.py b/examples/nlp/asr_postprocessor/asr_postprocessor.py index 047a32f2a94c..c9cfadd1ef73 100644 --- a/examples/nlp/asr_postprocessor/asr_postprocessor.py +++ b/examples/nlp/asr_postprocessor/asr_postprocessor.py @@ -113,7 +113,7 @@ args.d_model, num_classes=vocab_size, num_layers=1, log_softmax=True ) -loss_fn = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM(pad_id=tokenizer.pad_id(), label_smoothing=0.1) +loss_fn = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM(pad_id=tokenizer.pad_id, label_smoothing=0.1) beam_search = nemo_nlp.nm.trainables.BeamSearchTranslatorNM( decoder=decoder, @@ -121,9 +121,9 @@ max_seq_length=args.max_seq_length, beam_size=args.beam_size, length_penalty=args.len_pen, - bos_token=tokenizer.bos_id(), - pad_token=tokenizer.pad_id(), - eos_token=tokenizer.eos_id(), + bos_token=tokenizer.bos_id, + pad_token=tokenizer.pad_id, + eos_token=tokenizer.eos_id, ) # tie all embeddings weights diff --git a/examples/nlp/language_modeling/language_modeling_transformer.py b/examples/nlp/language_modeling/language_modeling_transformer.py index 7400e18513e3..d49040949538 100644 --- a/examples/nlp/language_modeling/language_modeling_transformer.py +++ b/examples/nlp/language_modeling/language_modeling_transformer.py @@ -110,7 +110,7 @@ ) loss = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM( - pad_id=tokenizer.pad_id(), label_smoothing=args.label_smoothing + pad_id=tokenizer.pad_id, label_smoothing=args.label_smoothing ) # tie weight of embedding and log_softmax layers diff --git a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py index 3a214340ef2e..f8467eff6d51 100644 --- a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py +++ b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py @@ -155,13 +155,13 @@ log_softmax=log_softmax, max_seq_length=args.max_seq_length, beam_size=args.beam_size, - bos_token=tgt_tokenizer.bos_id(), - pad_token=tgt_tokenizer.pad_id(), - eos_token=tgt_tokenizer.eos_id(), + bos_token=tgt_tokenizer.bos_id, + pad_token=tgt_tokenizer.pad_id, + eos_token=tgt_tokenizer.eos_id, ) loss_fn = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM( - pad_id=tgt_tokenizer.pad_id(), label_smoothing=args.label_smoothing + pad_id=tgt_tokenizer.pad_id, label_smoothing=args.label_smoothing ) if tie_weight: @@ -261,7 +261,7 @@ def translate_sentence(text): output its translation """ ids = src_tokenizer.text_to_ids(text) - ids = [src_tokenizer.bos_id()] + ids + [src_tokenizer.eos_id()] + ids = [src_tokenizer.bos_id] + ids + [src_tokenizer.eos_id] ids_tensor = torch.Tensor(ids).long().to(encoder._device).unsqueeze(0) ids_mask = torch.ones_like(ids_tensor) encoder_states = encoder.forward(ids_tensor, ids_mask) diff --git a/nemo/collections/nlp/callbacks/machine_translation_callback.py b/nemo/collections/nlp/callbacks/machine_translation_callback.py index 73a7256e7893..cb38a26a3848 100644 --- a/nemo/collections/nlp/callbacks/machine_translation_callback.py +++ b/nemo/collections/nlp/callbacks/machine_translation_callback.py @@ -43,7 +43,7 @@ def eval_iter_callback(tensors, global_vars, tgt_tokenizer): if "tgt" in kv: ref = [] for tgt in v: - nonpad_tokens = (tgt != tgt_tokenizer.pad_id()).sum().item() + nonpad_tokens = (tgt != tgt_tokenizer.pad_id).sum().item() tgt_sentences = tgt.cpu().numpy().tolist() for sentence in tgt_sentences: ref.append(tgt_tokenizer.ids_to_text(sentence)) diff --git a/nemo/collections/nlp/data/datasets/lm_transformer_dataset.py b/nemo/collections/nlp/data/datasets/lm_transformer_dataset.py index 10bdb0a2b402..5d8f20723c6e 100644 --- a/nemo/collections/nlp/data/datasets/lm_transformer_dataset.py +++ b/nemo/collections/nlp/data/datasets/lm_transformer_dataset.py @@ -48,7 +48,7 @@ def __getitem__(self, idx): right = left + self.max_seq_length src_ids = self.ids[left:right] labels = self.ids[left + 1 : right + 1] - src_mask = (src_ids != self.tokenizer.pad_id()).astype(np.float32) + src_mask = (src_ids != self.tokenizer.pad_id).astype(np.float32) return src_ids, src_mask, labels @@ -147,7 +147,7 @@ def dataset_to_ids(dataset, tokenizer, cache_ids=False, add_bos_eos=True): for sentence in data: sent_ids = tokenizer.text_to_ids(sentence.decode("utf-8")) if add_bos_eos: - sent_ids = [tokenizer.bos_id()] + sent_ids + [tokenizer.eos_id()] + sent_ids = [tokenizer.bos_id] + sent_ids + [tokenizer.eos_id] ids.append(sent_ids) if cache_ids: logging.info("Caching tokenized dataset ...") diff --git a/nemo/collections/nlp/data/datasets/machine_translation_dataset.py b/nemo/collections/nlp/data/datasets/machine_translation_dataset.py index b37e19563731..db8e6b7ace2d 100644 --- a/nemo/collections/nlp/data/datasets/machine_translation_dataset.py +++ b/nemo/collections/nlp/data/datasets/machine_translation_dataset.py @@ -48,8 +48,8 @@ def __getitem__(self, idx): tgt = self.batches[idx]["tgt"] labels = tgt[:, 1:] tgt_ids = tgt[:, :-1] - src_mask = (src_ids != self.src_tokenizer.pad_id()).astype(np.int32) - tgt_mask = (tgt_ids != self.tgt_tokenizer.pad_id()).astype(np.int32) + src_mask = (src_ids != self.src_tokenizer.pad_id).astype(np.int32) + tgt_mask = (tgt_ids != self.tgt_tokenizer.pad_id).astype(np.int32) sent_ids = self.batch_indices[idx] return src_ids, src_mask, tgt_ids, tgt_mask, labels, sent_ids @@ -63,8 +63,8 @@ def pad_batches(self, src_ids, tgt_ids, batch_indices): for batch_idx, b in enumerate(batch_indices): src_len = max([len(src_ids[i]) for i in b]) tgt_len = max([len(tgt_ids[i]) for i in b]) - src_ids_ = self.src_tokenizer.pad_id() * np.ones((len(b), src_len), dtype=np.int) - tgt_ids_ = self.tgt_tokenizer.pad_id() * np.ones((len(b), tgt_len), dtype=np.int) + src_ids_ = self.src_tokenizer.pad_id * np.ones((len(b), src_len), dtype=np.int) + tgt_ids_ = self.tgt_tokenizer.pad_id * np.ones((len(b), tgt_len), dtype=np.int) for i, sentence_idx in enumerate(b): src_ids_[i][: len(src_ids[sentence_idx])] = src_ids[sentence_idx] tgt_ids_[i][: len(tgt_ids[sentence_idx])] = tgt_ids[sentence_idx] diff --git a/nemo/collections/nlp/data/tokenizers/char_tokenizer.py b/nemo/collections/nlp/data/tokenizers/char_tokenizer.py index 0224ba39c842..f654f8f53e14 100644 --- a/nemo/collections/nlp/data/tokenizers/char_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/char_tokenizer.py @@ -57,11 +57,14 @@ def tokens_to_ids(self, tokens): def ids_to_tokens(self, ids): return [self.inv_vocab[id] for id in ids] + @property def pad_id(self): return self.vocab[""] + @property def bos_id(self): return self.vocab[""] + @property def eos_id(self): return self.vocab[""] diff --git a/nemo/collections/nlp/data/tokenizers/gpt2_tokenizer.py b/nemo/collections/nlp/data/tokenizers/gpt2_tokenizer.py index cad9b78b1058..945211ed9604 100644 --- a/nemo/collections/nlp/data/tokenizers/gpt2_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/gpt2_tokenizer.py @@ -72,11 +72,14 @@ def ids_to_text(self, ids): text = self.tokens_to_text(tokens) return text + @property def pad_id(self): return self.tokens_to_ids([self.tokenizer.pad_token])[0] + @property def bos_id(self): return self.tokens_to_ids([self.tokenizer.bos_token])[0] + @property def eos_id(self): return self.tokens_to_ids([self.tokenizer.eos_token])[0] diff --git a/nemo/collections/nlp/data/tokenizers/word_tokenizer.py b/nemo/collections/nlp/data/tokenizers/word_tokenizer.py index 4b47b508abcc..90421727ffb9 100644 --- a/nemo/collections/nlp/data/tokenizers/word_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/word_tokenizer.py @@ -57,11 +57,14 @@ def tokens_to_ids(self, tokens): def ids_to_tokens(self, ids): return [self.inv_vocab[id] for id in ids] + @property def pad_id(self): return self.vocab[""] + @property def bos_id(self): return self.vocab[""] + @property def eos_id(self): return self.vocab[""] diff --git a/nemo/collections/nlp/data/tokenizers/youtokentome_tokenizer.py b/nemo/collections/nlp/data/tokenizers/youtokentome_tokenizer.py index 0381126290b0..30778cebc474 100644 --- a/nemo/collections/nlp/data/tokenizers/youtokentome_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/youtokentome_tokenizer.py @@ -47,11 +47,14 @@ def ids_to_tokens(self, ids): ids_ = [id_ for id_ in ids if id_ not in self.special_tokens] return [self.tokenizer.id_to_subword(id_) for id_ in ids_] + @property def pad_id(self): return self.tokenizer.subword_to_id("") + @property def bos_id(self): return self.tokenizer.subword_to_id("") + @property def eos_id(self): return self.tokenizer.subword_to_id("") From 4b3210cce2f26ffd4fb51695b7a76588056aebc0 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 16:26:14 -0800 Subject: [PATCH 42/70] add ci test for roberta, reduce ci for asr_postprocessor Signed-off-by: Yang Zhang --- Jenkinsfile | 16 ++++++++++++---- .../nlp/language_modeling/bert_pretraining.py | 6 ++---- .../question_answering_squad.py | 8 ++++---- .../nlp/data/tokenizers/bert_tokenizer.py | 3 ++- .../nlp/nm/data_layers/qa_squad_datalayer.py | 2 +- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0829b8602f32..db77c4605399 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,14 +118,14 @@ pipeline { stage('Parallel NLP-Squad') { failFast true parallel { - stage('Squad v1.1') { + stage('BERT Squad v1.1') { steps { sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=0 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case' sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv1/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/question_answering/outputs/squadv1 && rm -rf /home/mrjenkins/TestData/nlp/squad_mini/v1.1/*cache*' } } - stage('Squad v2.0') { + stage('BERT Squad v2.0') { steps { sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v2.0/train-v2.0.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v2.0/dev-v2.0.json --work_dir outputs/squadv2 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --version_2_with_negative' sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv2/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' @@ -135,16 +135,24 @@ pipeline { } } - stage('NLP-ASR processing') { + + stage('Parallel NLP-Examples 3') { failFast true parallel { stage('asr_processing') { steps { - sh 'cd examples/nlp/asr_postprocessor && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 asr_postprocessor.py --data_dir=/home/mrjenkins/TestData/nlp/asr_postprocessor/pred_real --restore_from=/home/mrjenkins/TestData/nlp/asr_postprocessor/bert-base-uncased_decoder.pt --max_steps=50 --batch_size=512' + sh 'cd examples/nlp/asr_postprocessor && CUDA_VISIBLE_DEVICES=0 python asr_postprocessor.py --data_dir=/home/mrjenkins/TestData/nlp/asr_postprocessor/pred_real --restore_from=/home/mrjenkins/TestData/nlp/asr_postprocessor/bert-base-uncased_decoder.pt --max_steps=25 --batch_size=64' sh 'cd examples/nlp/asr_postprocessor && WER=$(cat outputs/asr_postprocessor/log_globalrank-0_localrank-0.txt | grep "Validation WER" | tail -n 1 | egrep -o "[0-9.]+" | tail -n 1) && echo $WER && if [ $(echo "$WER < 2.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/asr_postprocessor/outputs' } } + stage('Roberta Squad v1.1') { + steps { + sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1_roberta --batch_size 2 --save_step_freq 500 --num_epochs 1 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --model_type roberta --pretrained_model_name roberta-base' + sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv1_roberta/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' + sh 'rm -rf examples/nlp/question_answering/outputs/squadv1_roberta && rm -rf /home/mrjenkins/TestData/nlp/squad_mini/v1.1/*cache*' + } + } } } diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 428d2e5cb1c7..5b4658a2f61d 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -178,11 +178,9 @@ ) if args.tokenizer == "sentence-piece": logging.info("To use SentencePieceTokenizer.") - print(nemo_nlp.__file__) tokenizer = nemo_nlp.data.SentencePieceTokenizer( model_path=data_desc.tokenizer_model, special_tokens=special_tokens ) - # import ipdb; ipdb.set_trace() elif args.tokenizer == "nemo-bert": logging.info("To use NemoBertTokenizer.") vocab_file = os.path.join(args.data_dir, 'vocab.txt') @@ -192,7 +190,7 @@ raise ValueError("Please add your tokenizer " "or use sentence-piece or nemo-bert.") args.vocab_size = tokenizer.vocab_size -print(vars(args)) +(vars(args)) bert_model = nemo_nlp.nm.trainables.huggingface.BERT( vocab_size=args.vocab_size, num_hidden_layers=args.num_hidden_layers, @@ -281,7 +279,7 @@ def create_pipeline(data_file, batch_size, preprocessed_data=False, batches_per_ batches_per_step=args.batches_per_step, ) -print("steps per epoch", steps_per_epoch) +logging.info("steps per epoch", steps_per_epoch) # callback which prints training loss and perplexity once in a while if not args.only_mlm_loss: log_tensors = [train_loss, mlm_loss, nsp_loss] diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index ffe87f08abc9..17bccbf5cac6 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -81,7 +81,7 @@ def parse_args(): parser.add_argument( "--dev_file", type=str, required=True, help="The evaluation data file. Should be *.json", ) - parser.add_argument("--pretrained_bert_model", type=str, help="Name of the pre-trained model") + parser.add_argument("--pretrained_model_name", type=str, help="Name of the pre-trained model") parser.add_argument("--checkpoint_dir", default=None, type=str, help="Checkpoint directory for inference.") parser.add_argument( "--bert_checkpoint", default=None, type=str, help="Path to BERT model checkpoint for finetuning." @@ -337,8 +337,8 @@ def create_pipeline( model_name = MODEL_CLASSES[args.model_type]["model_name"] tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] - if args.pretrained_bert_model is None: - args.pretrained_bert_model = model_name + if args.pretrained_model_name is None: + args.pretrained_model_name = model_name tokenizer = tokenizer_cls( do_lower_case=args.do_lower_case, @@ -356,7 +356,7 @@ def create_pipeline( To see the list of pretrained models, call: nemo_nlp.huggingface.BERT.list_pretrained_models() """ - model = model_cls(pretrained_model_name=args.pretrained_bert_model) + model = model_cls(pretrained_model_name=args.pretrained_model_name) hidden_size = model.hidden_size diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index c52692f9dcae..4bce6f9ced7a 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -86,7 +86,8 @@ def __init__( tokenizer_cls = RobertaTokenizer self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) - self.vocab_size = len(self.tokenizer.vocab) + if hasattr(self.tokenizer, "vocab"): + self.vocab_size = len(self.tokenizer.vocab) for k, v in special_tokens.items(): setattr(self, k, v) diff --git a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py index 3d18df1a81c6..2cf983ac002a 100644 --- a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py @@ -26,7 +26,7 @@ class BertQuestionAnsweringDataLayer(TextDataLayer): Creates the data layer to use for Question Answering classification task. Args: - data_file (str): Directory that contains train.*.json and dev.*.json. + data_file (str): data_file in *.json. tokenizer (obj): Tokenizer object, e.g. NemoBertTokenizer. version_2_with_negative (bool): True if training should allow unanswerable questions. From 4667cdb095df36952d5e000503ab475c336eb800 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 16:43:51 -0800 Subject: [PATCH 43/70] fix test threshhold Signed-off-by: Yang Zhang --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index db77c4605399..4109d8eb6597 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,7 +142,7 @@ pipeline { stage('asr_processing') { steps { sh 'cd examples/nlp/asr_postprocessor && CUDA_VISIBLE_DEVICES=0 python asr_postprocessor.py --data_dir=/home/mrjenkins/TestData/nlp/asr_postprocessor/pred_real --restore_from=/home/mrjenkins/TestData/nlp/asr_postprocessor/bert-base-uncased_decoder.pt --max_steps=25 --batch_size=64' - sh 'cd examples/nlp/asr_postprocessor && WER=$(cat outputs/asr_postprocessor/log_globalrank-0_localrank-0.txt | grep "Validation WER" | tail -n 1 | egrep -o "[0-9.]+" | tail -n 1) && echo $WER && if [ $(echo "$WER < 2.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' + sh 'cd examples/nlp/asr_postprocessor && WER=$(cat outputs/asr_postprocessor/log_globalrank-0_localrank-0.txt | grep "Validation WER" | tail -n 1 | egrep -o "[0-9.]+" | tail -n 1) && echo $WER && if [ $(echo "$WER < 25.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/asr_postprocessor/outputs' } } From 5ddb513c353af6e422491033f4df0061ec30f130 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 17:01:03 -0800 Subject: [PATCH 44/70] fix chatbot example Signed-off-by: Oleksii Kuchaiev --- examples/start_here/chatbot_example.py | 4 +++- nemo/backends/pytorch/tutorials/chatbot/modules.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/start_here/chatbot_example.py b/examples/start_here/chatbot_example.py index c5107411525d..ca2950c22bce 100644 --- a/examples/start_here/chatbot_example.py +++ b/examples/start_here/chatbot_example.py @@ -65,10 +65,12 @@ def outputs2words(tensors, vocab): tensors=[loss, src, outputs_inf, tgt], print_func=lambda x: outputs2words(x, dl.voc.index2word), ) +num_epochs = 1 +logging.info(f"Training only for {num_epochs}. Train longer (~10-20) for convergence.") # Start training nf.train( tensors_to_optimize=[loss], callbacks=[callback], optimizer="adam", - optimization_params={"num_epochs": 3, "lr": 0.001}, + optimization_params={"num_epochs": num_epochs, "lr": 0.001}, ) diff --git a/nemo/backends/pytorch/tutorials/chatbot/modules.py b/nemo/backends/pytorch/tutorials/chatbot/modules.py index ca37d874ec52..5d51697922aa 100644 --- a/nemo/backends/pytorch/tutorials/chatbot/modules.py +++ b/nemo/backends/pytorch/tutorials/chatbot/modules.py @@ -23,7 +23,7 @@ def output_ports(self): """ return { "src": NeuralType(ChannelType(), ('T', 'B')), - "src_lengths": NeuralType(ChannelType(), tuple('B')), + "src_lengths": NeuralType(LengthsType(), tuple('B')), "tgt": NeuralType(LabelsType(), ('T', 'B')), "mask": NeuralType(ChannelType(), ('T', 'B')), "max_tgt_lengths": NeuralType(axes=None), From a734d130be67d86e20f17e8f90702b788cffca2b Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 10:59:06 -0800 Subject: [PATCH 45/70] fix lgtm, add vocab as argument to nemoberttokenizer Signed-off-by: Yang Zhang --- examples/nlp/language_modeling/bert_pretraining.py | 3 +-- nemo/collections/nlp/data/tokenizers/bert_tokenizer.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 5b4658a2f61d..24b94f1fb380 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -183,9 +183,8 @@ ) elif args.tokenizer == "nemo-bert": logging.info("To use NemoBertTokenizer.") - vocab_file = os.path.join(args.data_dir, 'vocab.txt') # To train on a Chinese dataset, use NemoBertTokenizer - tokenizer = nemo_nlp.data.NemoBertTokenizer(vocab_file=vocab_file) + tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model="bert-base-uncased") else: raise ValueError("Please add your tokenizer " "or use sentence-piece or nemo-bert.") args.vocab_size = tokenizer.vocab_size diff --git a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py index 4bce6f9ced7a..61f86c5d41a3 100644 --- a/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/bert_tokenizer.py @@ -64,7 +64,8 @@ def remove_spaces(text): class NemoBertTokenizer(TokenizerSpec): def __init__( self, - pretrained_model, + pretrained_model=None, + vocab_file=None, bert_derivate='bert', special_tokens={ "unk_token": "[UNK]", @@ -84,8 +85,13 @@ def __init__( tokenizer_cls = AlbertTokenizer elif bert_derivate == 'roberta': tokenizer_cls = RobertaTokenizer + if pretrained_model is not None: + self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) + elif vocab_file is not None: + self.tokenizer = tokenizer_cls(vocab_file=vocab_file) + else: + raise ValueError("either 'vocab_file' or 'pretrained_model' has to be specified") - self.tokenizer = tokenizer_cls.from_pretrained(pretrained_model) if hasattr(self.tokenizer, "vocab"): self.vocab_size = len(self.tokenizer.vocab) for k, v in special_tokens.items(): From 9b0e3098fce03efefaad9ad04b2d1482dc52ef04 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 12 Feb 2020 11:39:23 -0800 Subject: [PATCH 46/70] update GAN dataset path to avoid download; merge GAN and Jasper testing to reduce test time Signed-off-by: Jason --- Jenkinsfile | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b6ec2d321979..0685082f93b3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -164,12 +164,17 @@ pipeline { } } - stage('Parallel Stage Jasper') { + stage('Parallel Stage Jasper / GAN') { failFast true parallel { - stage('Jasper AN4 O1') { + // stage('Jasper AN4 O1') { + // steps { + // sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0 python jasper_an4.py --amp_opt_level=O1 --num_epochs=35 --test_after_training --work_dir=O1' + // } + // } + stage('GAN O2') { steps { - sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0 python jasper_an4.py --amp_opt_level=O1 --num_epochs=35 --test_after_training --work_dir=O1' + sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O2 --num_epochs=3 --train_dataset=/home/mrjenkins/TestData/MNIST/' } } stage('Jasper AN4 O2') { @@ -180,21 +185,16 @@ pipeline { } } - stage('Parallel Stage GAN') { - failFast true - parallel { - stage('GAN O1') { - steps { - sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O1 --num_epochs=3' - } - } - stage('GAN O2') { - steps { - sh 'cd examples/image && CUDA_VISIBLE_DEVICES=1 python gan.py --amp_opt_level=O2 --num_epochs=3' - } - } - } - } + // stage('Parallel Stage GAN') { + // failFast true + // parallel { + // stage('GAN O1') { + // steps { + // sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O1 --num_epochs=3' + // } + // } + // } + // } stage('Multi-GPU test') { failFast true From ecbec6c159dbfef7d1bc2b0732752908e76dca08 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 11:51:20 -0800 Subject: [PATCH 47/70] addressing some review feedback Signed-off-by: Oleksii Kuchaiev --- nemo/backends/pytorch/common/losses.py | 10 +- nemo/backends/pytorch/common/other.py | 6 +- nemo/backends/pytorch/common/rnn.py | 16 +-- nemo/backends/pytorch/common/search.py | 6 +- .../pytorch/torchvision/data/image_folder.py | 14 +-- .../pytorch/tutorials/chatbot/modules.py | 38 +++--- nemo/backends/pytorch/tutorials/toys.py | 32 ++--- nemo/collections/asr/audio_preprocessing.py | 50 ++++---- nemo/collections/asr/beam_search_decoder.py | 6 +- nemo/collections/asr/data_layer.py | 18 +-- nemo/collections/asr/greedy_ctc_decoder.py | 4 +- nemo/collections/asr/jasper.py | 12 +- nemo/collections/asr/las/misc.py | 4 +- nemo/collections/asr/losses.py | 10 +- .../data_layers/glue_benchmark_datalayer.py | 16 +-- .../joint_intent_slot_datalayer.py | 24 ++-- .../nlp/nm/data_layers/lm_bert_datalayer.py | 24 ++-- .../data_layers/lm_transformer_datalayer.py | 6 +- .../machine_translation_datalayer.py | 12 +- .../punctuation_capitalization_datalayer.py | 14 +-- .../nlp/nm/data_layers/qa_squad_datalayer.py | 12 +- .../state_tracking_trade_datalayer.py | 10 +- .../text_classification_datalayer.py | 8 +- .../token_classification_datalayer.py | 22 ++-- .../nlp/nm/losses/aggregator_loss.py | 2 +- .../nlp/nm/losses/joint_intent_slot_loss.py | 12 +- .../losses/masked_language_modeling_loss.py | 8 +- .../padded_smoothed_cross_entropy_loss.py | 6 +- .../nlp/nm/losses/qa_squad_loss.py | 12 +- .../nm/losses/state_tracking_trade_loss.py | 14 +-- .../nm/losses/token_classification_loss.py | 8 +- .../trainables/common/huggingface/bert_nm.py | 8 +- .../common/sequence_classification_nm.py | 4 +- .../common/sequence_regression_nm.py | 4 +- .../common/token_classification_nm.py | 8 +- .../common/transformer/transformer_nm.py | 26 ++--- .../state_tracking_trade_nm.py | 14 +-- .../joint_intent_slot/joint_intent_slot_nm.py | 6 +- nemo/collections/simple_gan/gan.py | 32 ++--- nemo/collections/tts/data_layers.py | 4 +- nemo/collections/tts/tacotron2_modules.py | 60 +++++----- nemo/collections/tts/waveglow_modules.py | 22 ++-- nemo/core/neural_factory.py | 28 ++--- nemo/core/neural_modules.py | 1 + nemo/core/neural_types/axes.py | 13 ++- nemo/core/neural_types/elements.py | 82 ++++++------- nemo/core/neural_types/neural_type.py | 26 +++-- tests/asr/test_zeroDS.py | 9 +- tests/core/test_infer.py | 12 +- tests/core/test_neural_modules.py | 5 +- tests/core/test_neural_modules_pytorch.py | 5 +- tests/core/test_neural_types.py | 109 ++++++++---------- 52 files changed, 453 insertions(+), 461 deletions(-) diff --git a/nemo/backends/pytorch/common/losses.py b/nemo/backends/pytorch/common/losses.py index 60b091802c68..9d14f763e22d 100644 --- a/nemo/backends/pytorch/common/losses.py +++ b/nemo/backends/pytorch/common/losses.py @@ -45,7 +45,7 @@ def output_ports(self): NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__( self, pad_id=0, smoothing_coef=0.0, sample_wise=False, aux_ctc=False, ctc_initial_coef=0.1, ctc_blank_id=None @@ -107,8 +107,8 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "logits": NeuralType(elements_type=LogitsType(), axes=('B', 'D')), - "labels": NeuralType(elements_type=LabelsType(), axes=tuple('B')), + "logits": NeuralType(axes=('B', 'D'), elements_type=LogitsType()), + "labels": NeuralType(axes=tuple('B'), elements_type=LabelsType()), } @property @@ -143,8 +143,8 @@ def input_ports(self): 0: AxisType(RegressionTag) """ return { - "preds": NeuralType(RegressionValuesType(), tuple('B')), - "labels": NeuralType(LabelsType(), tuple('B')), + "preds": NeuralType(tuple('B'), RegressionValuesType()), + "labels": NeuralType(tuple('B'), LabelsType()), } @property diff --git a/nemo/backends/pytorch/common/other.py b/nemo/backends/pytorch/common/other.py index 7de337619f01..c9b9040dd32c 100644 --- a/nemo/backends/pytorch/common/other.py +++ b/nemo/backends/pytorch/common/other.py @@ -28,7 +28,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"outputs": NeuralType({0: AxisType(TimeTag), 1: AxisType(BatchTag), 2: AxisType(ChannelTag),})} - return {"outputs": NeuralType(ChannelType(), ('T', 'B', 'D'))} + return {"outputs": NeuralType(('T', 'B', 'D'), ChannelType())} def __init__(self, voc_size, hidden_size, dropout=0.0): super().__init__() @@ -53,14 +53,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} - return {"input_type_ids": NeuralType(VoidType(), ('B', 'T'))} + return {"input_type_ids": NeuralType(('B', 'T'), VoidType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag),})} - return {"input_type_ids": NeuralType(ChannelType(), ('B', 'T'))} + return {"input_type_ids": NeuralType(('B', 'T'), ChannelType())} def __init__(self): super().__init__() diff --git a/nemo/backends/pytorch/common/rnn.py b/nemo/backends/pytorch/common/rnn.py index 774e11807edf..fbf7dbb7eb97 100644 --- a/nemo/backends/pytorch/common/rnn.py +++ b/nemo/backends/pytorch/common/rnn.py @@ -70,11 +70,11 @@ def input_ports(self): """ return { # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - 'targets': NeuralType(LabelsType(), ('B', 'T')), + 'targets': NeuralType(('B', 'T'), LabelsType()), # 'encoder_outputs': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, # ), - 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'D'), True), + 'encoder_outputs': NeuralType(('B', 'T', 'D'), ChannelType(), True), } @property @@ -83,11 +83,11 @@ def output_ports(self): """ return { # 'log_probs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), - 'log_probs': NeuralType(LogprobsType(), ('B', 'T', 'D')), + 'log_probs': NeuralType(('B', 'T', 'D'), LogprobsType()), # 'attention_weights': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}, optional=True, # ), - 'attention_weights': NeuralType(ChannelType(), ('B', 'T', 'T'), True), + 'attention_weights': NeuralType(('B', 'T', 'T'), ChannelType(), True), } def __init__( @@ -209,8 +209,8 @@ def input_ports(self): return { # 'inputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'input_lens': NeuralType({0: AxisType(BatchTag),}, optional=True), - 'inputs': NeuralType(ChannelType(), ('B', 'T')), - 'input_lens': NeuralType(LengthsType(), tuple('B')), + 'inputs': NeuralType(('B', 'T'), ChannelType()), + 'input_lens': NeuralType(tuple('B'), LengthsType()), } @property @@ -220,8 +220,8 @@ def output_ports(self): return { # 'outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # 'hidden': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - 'outputs': NeuralType(ChannelType(), ('B', 'T', 'D')), - 'hidden': NeuralType(ChannelType(), ('B', 'T', 'D')), + 'outputs': NeuralType(('B', 'T', 'D'), ChannelType()), + 'hidden': NeuralType(('B', 'T', 'D'), ChannelType()), } def __init__( diff --git a/nemo/backends/pytorch/common/search.py b/nemo/backends/pytorch/common/search.py index 2051a648b6cb..acaf32213016 100644 --- a/nemo/backends/pytorch/common/search.py +++ b/nemo/backends/pytorch/common/search.py @@ -36,7 +36,7 @@ def input_ports(self): # 'encoder_outputs': NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}, optional=True, # ) - "encoder_outputs": NeuralType(ChannelType(), ('B', 'T', 'D'), optional=True) + "encoder_outputs": NeuralType(('B', 'T', 'D'), ChannelType(), optional=True) } @property @@ -47,8 +47,8 @@ def output_ports(self): return { # 'predictions': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'attention_weights': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), - "predictions": NeuralType(ChannelType(), ('B', 'T')), - "attention_weights": NeuralType(ChannelType(), ('B', 'T', 'T')), + "predictions": NeuralType(('B', 'T'), ChannelType()), + "attention_weights": NeuralType(('B', 'T', 'T'), ChannelType()), } def __init__(self, decoder, pad_id, bos_id, eos_id, max_len, batch_size=None): diff --git a/nemo/backends/pytorch/torchvision/data/image_folder.py b/nemo/backends/pytorch/torchvision/data/image_folder.py index 5c4946b5cdd5..b775efb1a8f5 100644 --- a/nemo/backends/pytorch/torchvision/data/image_folder.py +++ b/nemo/backends/pytorch/torchvision/data/image_folder.py @@ -27,14 +27,12 @@ def output_ports(self): 0: AxisType(BatchTag) """ return { - "image": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, self._input_size), - 3: AxisType(WidthTag, self._input_size), - } - ), + "image": NeuralType({ + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, self._input_size), + 3: AxisType(WidthTag, self._input_size), + }), "label": NeuralType({0: AxisType(BatchTag)}), } diff --git a/nemo/backends/pytorch/tutorials/chatbot/modules.py b/nemo/backends/pytorch/tutorials/chatbot/modules.py index 5d51697922aa..14d704b4d4fc 100644 --- a/nemo/backends/pytorch/tutorials/chatbot/modules.py +++ b/nemo/backends/pytorch/tutorials/chatbot/modules.py @@ -22,10 +22,10 @@ def output_ports(self): """Returns definitions of module output ports. """ return { - "src": NeuralType(ChannelType(), ('T', 'B')), - "src_lengths": NeuralType(LengthsType(), tuple('B')), - "tgt": NeuralType(LabelsType(), ('T', 'B')), - "mask": NeuralType(ChannelType(), ('T', 'B')), + "src": NeuralType(('T', 'B'), ChannelType()), + "src_lengths": NeuralType(tuple('B'), LengthsType()), + "tgt": NeuralType(('T', 'B'), LabelsType()), + "mask": NeuralType(('T', 'B'), ChannelType()), "max_tgt_lengths": NeuralType(axes=None), } @@ -75,8 +75,8 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "input_seq": NeuralType(ChannelType(), ('T', 'B')), - "input_lengths": NeuralType(LengthsType(), tuple('B')), + "input_seq": NeuralType(('T', 'B'), ChannelType()), + "input_lengths": NeuralType(tuple('B'), LengthsType()), } @property @@ -84,8 +84,8 @@ def output_ports(self): """Returns definitions of module output ports. """ return { - "outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), - "hidden": NeuralType(ChannelType(), ('B', 'D')), + "outputs": NeuralType(('T', 'B', 'D'), ChannelType()), + "hidden": NeuralType(('B', 'D'), ChannelType()), } def __init__(self, voc_size, encoder_n_layers, hidden_size, dropout, bidirectional=True): @@ -135,8 +135,8 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "targets": NeuralType(LabelsType(), ('T', 'B')), - "encoder_outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), + "targets": NeuralType(('T', 'B'), LabelsType()), + "encoder_outputs": NeuralType(('T', 'B', 'D'), ChannelType()), "max_target_len": NeuralType(axes=None), } @@ -157,8 +157,8 @@ def output_ports(self): 1: AxisType(ChannelTag) """ return { - "outputs": NeuralType(ChannelType(), ('T', 'B', 'D')), - "hidden": NeuralType(ChannelType(), ('B', 'D')), + "outputs": NeuralType(('T', 'B', 'D'), ChannelType()), + "hidden": NeuralType(('B', 'D'), ChannelType()), } def __init__(self, attn_model, hidden_size, voc_size, decoder_n_layers, dropout): @@ -273,9 +273,9 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "predictions": NeuralType(ChannelType(), ('T', 'B', 'D')), - "target": NeuralType(LabelsType(), ('T', 'B')), - "mask": NeuralType(ChannelType(), ('T', 'B')), + "predictions": NeuralType(('T', 'B', 'D'), ChannelType()), + "target": NeuralType(('T', 'B'), LabelsType()), + "mask": NeuralType(('T', 'B'), ChannelType()), } @property @@ -285,7 +285,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType(), axes=None)} + return {"loss": NeuralType(axes=None, elements_type=LossType())} def __init__(self): super().__init__() @@ -309,15 +309,15 @@ class GreedyLuongAttnDecoderRNN(TrainableNM): def input_ports(self): """Returns definitions of module input ports. """ - return {"encoder_outputs": NeuralType(ChannelType(), ('T', 'B', 'D'))} + return {"encoder_outputs": NeuralType(('T', 'B', 'D'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ return { - "outputs": NeuralType(ChannelType(), ('T', 'B')), - "hidden": NeuralType(ChannelType(), ('B', 'D')), + "outputs": NeuralType(('T', 'B'), ChannelType()), + "hidden": NeuralType(('B', 'D'), ChannelType()), } def __init__(self, attn_model, hidden_size, voc_size, decoder_n_layers, dropout, max_dec_steps=10): diff --git a/nemo/backends/pytorch/tutorials/toys.py b/nemo/backends/pytorch/tutorials/toys.py index 8b18d9890c60..442c841ee836 100644 --- a/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/backends/pytorch/tutorials/toys.py @@ -21,7 +21,7 @@ def input_ports(self): Returns: A (dict) of module's input ports names to NeuralTypes mapping """ - return {"x": NeuralType(ChannelType(), ('B', 'D'))} + return {"x": NeuralType(('B', 'D'), ChannelType())} @property def output_ports(self): @@ -30,7 +30,7 @@ def output_ports(self): Returns: A (dict) of module's output ports names to NeuralTypes mapping """ - return {"y_pred": NeuralType(ChannelType(), ('B', 'D'))} + return {"y_pred": NeuralType(('B', 'D'), ChannelType())} def __init__(self, dim): # Part specific for Neural Modules API: @@ -63,15 +63,15 @@ def input_ports(self): """ return { - "x": NeuralType(ChannelType(), ('B', 'D')), - "o": NeuralType(ChannelType(), ('B', 'D')), + "x": NeuralType(('B', 'D'), ChannelType()), + "o": NeuralType(('B', 'D'), ChannelType()), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"y_pred": NeuralType(ChannelType(), ('B', 'D'), optional=True)} + return {"y_pred": NeuralType(('B', 'D'), ChannelType(), optional=True)} def __init__(self, dim): # Part specific for Neural Modules API: @@ -123,8 +123,8 @@ def output_ports(self): """Returns definitions of module output ports """ return { - "x": NeuralType(ChannelType(), ('B', 'D')), - "y": NeuralType(LabelsType(), ('B', 'D')), + "x": NeuralType(('B', 'D'), ChannelType()), + "y": NeuralType(('B', 'D'), LabelsType()), } def __init__(self, batch_size, f_name="sin", n=1000, x_lo=-4, x_hi=4): @@ -182,15 +182,15 @@ def input_ports(self): 1: AxisType(ChannelTag) """ return { - "predictions": NeuralType(ChannelType(), ('B', 'D')), - "target": NeuralType(LabelsType(), ('B', 'D')), + "predictions": NeuralType(('B', 'D'), ChannelType()), + "target": NeuralType(('B', 'D'), LabelsType()), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self): super().__init__() @@ -206,15 +206,15 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "predictions": NeuralType(ChannelType(), ('B', 'D')), - "target": NeuralType(LabelsType(), ('B', 'D')), + "predictions": NeuralType(('B', 'D'), ChannelType()), + "target": NeuralType(('B', 'D'), LabelsType()), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self): super().__init__() @@ -230,8 +230,8 @@ def input_ports(self): """Returns definitions of module input ports. """ return { - "predictions": NeuralType(ChannelType(), ('B', 'D')), - "labels": NeuralType(LabelsType(), tuple('B')), + "predictions": NeuralType(('B', 'D'), ChannelType()), + "labels": NeuralType(tuple('B'), LabelsType()), } @property @@ -241,7 +241,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self): # Neural Module API specific diff --git a/nemo/collections/asr/audio_preprocessing.py b/nemo/collections/asr/audio_preprocessing.py index f2950162a346..945f4383caac 100644 --- a/nemo/collections/asr/audio_preprocessing.py +++ b/nemo/collections/asr/audio_preprocessing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -125,8 +125,8 @@ def input_ports(self): return { # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), - "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')), + "input_signal": NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + "length": NeuralType(tuple('B'), LengthsType()), } @property @@ -138,8 +138,8 @@ def output_ports(self): # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), - "processed_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')), + "processed_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -200,6 +200,10 @@ def __init__( def get_features(self, input_signal, length): return self.featurizer(input_signal) + @property + def sample_rate(self): + return self._sample_rate + class AudioToMelSpectrogramPreprocessor(AudioPreprocessor): """Featurizer that converts wavs to mel spectrograms. @@ -271,8 +275,8 @@ def input_ports(self): return { # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), - "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')), + "input_signal": NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + "length": NeuralType(tuple('B'), LengthsType()), } @property @@ -297,8 +301,8 @@ def output_ports(self): # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), - "processed_signal": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')), + "processed_signal": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -417,8 +421,8 @@ def input_ports(self): return { # "input_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "length": NeuralType({0: AxisType(BatchTag)}), - "input_signal": NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - "length": NeuralType(LengthsType(), tuple('B')), + "input_signal": NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + "length": NeuralType(tuple('B'), LengthsType()), } @property @@ -430,8 +434,8 @@ def output_ports(self): # {0: AxisType(BatchTag), 1: AxisType(MFCCSignalTag), 2: AxisType(ProcessedTimeTag),} # ), # "processed_length": NeuralType({0: AxisType(BatchTag)}), - "processed_signal": NeuralType(MFCCSpectrogramType(), ('B', 'D', 'T')), - "processed_length": NeuralType(LengthsType(), tuple('B')), + "processed_signal": NeuralType(('B', 'D', 'T'), MFCCSpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -547,7 +551,7 @@ def input_ports(self): return { # "input_spec": NeuralType({0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType( # TimeTag),}) - "input_spec": NeuralType(SpectrogramType(), ('B', 'D', 'T')) + "input_spec": NeuralType(('B', 'D', 'T'), SpectrogramType()) } @property @@ -558,7 +562,7 @@ def output_ports(self): # "augmented_spec": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} # ) - "augmented_spec": NeuralType(SpectrogramType(), ('B', 'D', 'T')) + "augmented_spec": NeuralType(('B', 'D', 'T'), SpectrogramType()) } def __init__( @@ -612,10 +616,10 @@ def input_ports(self): # "in_x_len": NeuralType({0: AxisType(BatchTag)}), # "in_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "in_y_len": NeuralType({0: AxisType(BatchTag)}), - "in_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "in_x_len": NeuralType(LengthsType(), tuple('B')), - "in_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "in_y_len": NeuralType(LengthsType(), tuple('B')), + "in_x": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "in_x_len": NeuralType(tuple('B'), LengthsType()), + "in_y": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "in_y_len": NeuralType(tuple('B'), LengthsType()), } @property @@ -627,10 +631,10 @@ def output_ports(self): # "out_x_len": NeuralType({0: AxisType(BatchTag)}), # "out_y": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "out_y_len": NeuralType({0: AxisType(BatchTag)}), - "out_x": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "out_x_len": NeuralType(LengthsType(), tuple('B')), - "out_y": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "out_y_len": NeuralType(LengthsType(), tuple('B')), + "out_x": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "out_x_len": NeuralType(tuple('B'), LengthsType()), + "out_y": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "out_y_len": NeuralType(tuple('B'), LengthsType()), } def __init__(self, mult_batch=1): diff --git a/nemo/collections/asr/beam_search_decoder.py b/nemo/collections/asr/beam_search_decoder.py index 2cb919ee4fe4..ecebe7a00ec3 100644 --- a/nemo/collections/asr/beam_search_decoder.py +++ b/nemo/collections/asr/beam_search_decoder.py @@ -45,8 +45,8 @@ def input_ports(self): return { # "log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),}), # "log_probs_length": NeuralType({0: AxisType(BatchTag)}), - "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), - "log_probs_length": NeuralType(LengthsType(), tuple('B')), + "log_probs": NeuralType(('B', 'T', 'D'), LogprobsType()), + "log_probs_length": NeuralType(tuple('B'), LengthsType()), } @property @@ -57,7 +57,7 @@ def output_ports(self): NeuralType(None) """ # return {"predictions": NeuralType(VoidType())} - return {"predictions": NeuralType(PredictionsType(), ('B', 'T'))} + return {"predictions": NeuralType(('B', 'T'), PredictionsType())} def __init__(self, vocab, beam_width, alpha, beta, lm_path, num_cpus, cutoff_prob=1.0, cutoff_top_n=40): diff --git a/nemo/collections/asr/data_layer.py b/nemo/collections/asr/data_layer.py index 83d959a09974..e2b95c0e9604 100644 --- a/nemo/collections/asr/data_layer.py +++ b/nemo/collections/asr/data_layer.py @@ -100,10 +100,10 @@ def output_ports(self): # 'a_sig_length': NeuralType({0: AxisType(BatchTag)}), # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), - 'audio_signal': NeuralType(AudioSignal(freq=self._sample_rate), ('B', 'T')), - 'a_sig_length': NeuralType(LengthsType(), tuple('B')), - 'transcripts': NeuralType(LabelsType(), ('B', 'T')), - 'transcript_length': NeuralType(LengthsType(), tuple('B')), + 'audio_signal': NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + 'transcripts': NeuralType(('B', 'T'), LabelsType()), + 'transcript_length': NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -221,9 +221,9 @@ def output_ports(self): # 'processed_length': NeuralType({0: AxisType(BatchTag)}), # 'transcripts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'transcript_length': NeuralType({0: AxisType(BatchTag)}), - 'processed_signal': NeuralType(SpectrogramType(), ('B', 'D', 'T')), - 'transcripts': NeuralType(ChannelType(), ('B', 'T')), - 'transcript_length': NeuralType(LengthsType(), tuple('B')), + 'processed_signal': NeuralType(('B', 'D', 'T'), SpectrogramType()), + 'transcripts': NeuralType(('B', 'T'), ChannelType()), + 'transcript_length': NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -350,8 +350,8 @@ def output_ports(self): return { # 'texts': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'texts_length': NeuralType({0: AxisType(BatchTag)}), - 'texts': NeuralType(ChannelType(), ('B', 'T')), - 'texts_length': NeuralType(LengthsType(), tuple('B')), + 'texts': NeuralType(('B', 'T'), ChannelType()), + 'texts_length': NeuralType(tuple('B'), LengthsType()), } def __init__( diff --git a/nemo/collections/asr/greedy_ctc_decoder.py b/nemo/collections/asr/greedy_ctc_decoder.py index 8f29ab9c3c40..2d49011e7235 100644 --- a/nemo/collections/asr/greedy_ctc_decoder.py +++ b/nemo/collections/asr/greedy_ctc_decoder.py @@ -15,14 +15,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"log_probs": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} - return {"log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D'))} + return {"log_probs": NeuralType(('B', 'T', 'D'), LogprobsType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"predictions": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"predictions": NeuralType(PredictionsType(), ('B', 'T'))} + return {"predictions": NeuralType(('B', 'T'), PredictionsType())} def __init__(self): super().__init__() diff --git a/nemo/collections/asr/jasper.py b/nemo/collections/asr/jasper.py index a1e41a8111b2..d6fcf7e38259 100644 --- a/nemo/collections/asr/jasper.py +++ b/nemo/collections/asr/jasper.py @@ -79,8 +79,8 @@ def input_ports(self): # {0: AxisType(BatchTag), 1: AxisType(SpectrogramSignalTag), 2: AxisType(ProcessedTimeTag),} # ), # "length": NeuralType({0: AxisType(BatchTag)}), - "audio_signal": NeuralType(SpectrogramType(), ('B', 'D', 'T')), - "length": NeuralType(LengthsType(), tuple('B')), + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), } @property @@ -92,8 +92,8 @@ def output_ports(self): # {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} # ), # "encoded_lengths": NeuralType({0: AxisType(BatchTag)}), - "outputs": NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')), - "encoded_lengths": NeuralType(LengthsType(), tuple('B')), + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), } def __init__( @@ -184,7 +184,7 @@ def input_ports(self): # "encoder_output": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(EncodedRepresentationTag), 2: AxisType(ProcessedTimeTag),} # ) - "encoder_output": NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')) + "encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()) } @property @@ -192,7 +192,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} - return {"output": NeuralType(LogprobsType(), ('B', 'T', 'D'))} + return {"output": NeuralType(('B', 'T', 'D'), LogprobsType())} def __init__(self, feat_in, num_classes, init_mode="xavier_uniform"): super().__init__() diff --git a/nemo/collections/asr/las/misc.py b/nemo/collections/asr/las/misc.py index 1ed2aadc5fb9..56519e143fd8 100644 --- a/nemo/collections/asr/las/misc.py +++ b/nemo/collections/asr/las/misc.py @@ -22,7 +22,7 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag),})} - return {'tensor': NeuralType(ChannelType(), ('B', 'D', 'T'))} + return {'tensor': NeuralType(('B', 'D', 'T'), ChannelType())} @property def output_ports(self): @@ -36,7 +36,7 @@ def output_ports(self): 2: AxisType(ChannelTag) """ # return {'tensor': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag),})} - return {'tensor': NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {'tensor': NeuralType(('B', 'T', 'D'), ChannelType())} def __init__(self, in_channels, out_channels): super().__init__() diff --git a/nemo/collections/asr/losses.py b/nemo/collections/asr/losses.py index a9b77fe03e0b..909a16d6f39c 100644 --- a/nemo/collections/asr/losses.py +++ b/nemo/collections/asr/losses.py @@ -24,10 +24,10 @@ def input_ports(self): # "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_length": NeuralType({0: AxisType(BatchTag)}), # "target_length": NeuralType({0: AxisType(BatchTag)}), - "log_probs": NeuralType(LogprobsType(), ('B', 'T', 'D')), - "targets": NeuralType(LabelsType(), ('B', 'T')), - "input_length": NeuralType(LengthsType(), tuple('B')), - "target_length": NeuralType(LengthsType(), tuple('B')), + "log_probs": NeuralType(('B', 'T', 'D'), LogprobsType()), + "targets": NeuralType(('B', 'T'), LabelsType()), + "input_length": NeuralType(tuple('B'), LengthsType()), + "target_length": NeuralType(tuple('B'), LengthsType()), } @property @@ -38,7 +38,7 @@ def output_ports(self): NeuralType(None) """ # return {"loss": NeuralType(None)} - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, num_classes): super().__init__() diff --git a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py index 95b89c7761e6..ac5ae86cca6c 100644 --- a/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/glue_benchmark_datalayer.py @@ -42,10 +42,10 @@ def output_ports(self): # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(CategoricalTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(CategoricalValuesType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(tuple('B'), CategoricalValuesType()), } def __init__( @@ -93,10 +93,10 @@ def output_ports(self): # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(RegressionTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(RegressionValuesType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(tuple('B'), RegressionValuesType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py index 187d859819f4..c306cfcccc04 100644 --- a/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/joint_intent_slot_datalayer.py @@ -52,13 +52,13 @@ def output_ports(self): # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "intents": NeuralType({0: AxisType(BatchTag)}), # "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), - "intents": NeuralType(ChannelType(), tuple('B')), - "slots": NeuralType(ChannelType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "subtokens_mask": NeuralType(('B', 'T'), ChannelType()), + "intents": NeuralType(tuple('B'), ChannelType()), + "slots": NeuralType(('B', 'T'), ChannelType()), } def __init__( @@ -118,11 +118,11 @@ def output_ports(self): # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "subtokens_mask": NeuralType(('B', 'T'), ChannelType()), } def __init__(self, queries, tokenizer, max_seq_length, batch_size=1, dataset_type=BertJointIntentSlotInferDataset): diff --git a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py index a6f8556529c4..98c1ba23c10f 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_bert_datalayer.py @@ -56,12 +56,12 @@ def output_ports(self): # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "output_ids": NeuralType(ChannelType(), ('B', 'T')), - "output_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "output_ids": NeuralType(('B', 'T'), ChannelType()), + "output_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(tuple('B'), LabelsType()), } def __init__(self, tokenizer, dataset, max_seq_length, mask_probability, short_seq_prob=0.1, batch_size=64): @@ -101,12 +101,12 @@ def output_ports(self): # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "output_ids": NeuralType(ChannelType(), ('B', 'T')), - "output_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "output_ids": NeuralType(('B', 'T'), ChannelType()), + "output_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(tuple('B'), LabelsType()), } def __init__(self, dataset, max_pred_length, batch_size=64, training=True): diff --git a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py index c04d8f8a1cb5..ebd1b2a738d0 100644 --- a/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/lm_transformer_datalayer.py @@ -58,9 +58,9 @@ def output_ports(self): # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(('B', 'T'), LabelsType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py index 53dcf0c7f0d9..44f877f5dcc3 100644 --- a/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/machine_translation_datalayer.py @@ -70,12 +70,12 @@ def output_ports(self): # "tgt_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "sent_ids": NeuralType({0: AxisType(BatchTag)}), - "src_ids": NeuralType(ChannelType(), ('B', 'T')), - "src_mask": NeuralType(ChannelType(), ('B', 'T')), - "tgt_ids": NeuralType(ChannelType(), ('B', 'T')), - "tgt_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), ('B', 'T')), - "sent_ids": NeuralType(ChannelType(), tuple('B')), + "src_ids": NeuralType(('B', 'T'), ChannelType()), + "src_mask": NeuralType(('B', 'T'), ChannelType()), + "tgt_ids": NeuralType(('B', 'T'), ChannelType()), + "tgt_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(('B', 'T'), LabelsType()), + "sent_ids": NeuralType(tuple('B'), ChannelType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py index b41501b5b684..e3cfeda2235a 100644 --- a/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/punctuation_capitalization_datalayer.py @@ -34,13 +34,13 @@ def output_ports(self): # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "punct_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "capit_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), - "punct_labels": NeuralType(LabelsType(), ('B', 'T')), - "capit_labels": NeuralType(LabelsType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "subtokens_mask": NeuralType(('B', 'T'), ChannelType()), + "punct_labels": NeuralType(('B', 'T'), LabelsType()), + "capit_labels": NeuralType(('B', 'T'), LabelsType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py index b61234462872..24ef5897fb1f 100644 --- a/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/qa_squad_datalayer.py @@ -56,12 +56,12 @@ def output_ports(self): # "start_positions": NeuralType({0: AxisType(BatchTag)}), # "end_positions": NeuralType({0: AxisType(BatchTag)}), # "unique_ids": NeuralType({0: AxisType(BatchTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "start_positions": NeuralType(ChannelType(), tuple('B')), - "end_positions": NeuralType(ChannelType(), tuple('B')), - "unique_ids": NeuralType(ChannelType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "start_positions": NeuralType(tuple('B'), ChannelType()), + "end_positions": NeuralType(tuple('B'), ChannelType()), + "unique_ids": NeuralType(tuple('B'), ChannelType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index a06562966591..8435dc976b8c 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -74,11 +74,11 @@ def output_ports(self): # "tgt_lens": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "gating_labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "turn_domain": NeuralType(None), - "src_ids": NeuralType(ChannelType(), ('B', 'T')), - "src_lens": NeuralType(LengthsType(), tuple('B')), - "tgt_ids": NeuralType(ChannelType(), ('B', 'D', 'T')), - "tgt_lens": NeuralType(LengthsType(), ('B', 'D')), - "gating_labels": NeuralType(LabelsType(), ('B', 'D')), + "src_ids": NeuralType(('B', 'T'), ChannelType()), + "src_lens": NeuralType(tuple('B'), LengthsType()), + "tgt_ids": NeuralType(('B', 'D', 'T'), ChannelType()), + "tgt_lens": NeuralType(('B', 'D'), LengthsType()), + "gating_labels": NeuralType(('B', 'D'), LabelsType()), "turn_domain": NeuralType(), } diff --git a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py index d35c5d401a56..a104a5a543f5 100644 --- a/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/text_classification_datalayer.py @@ -42,10 +42,10 @@ def output_ports(self): # "input_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), tuple('B')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(tuple('B'), LabelsType()), } def __init__( diff --git a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py index e1506aab5d2d..5fd6cbe2ee5b 100644 --- a/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/token_classification_datalayer.py @@ -33,12 +33,12 @@ def output_ports(self): # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), - "labels": NeuralType(LabelsType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "subtokens_mask": NeuralType(('B', 'T'), ChannelType()), + "labels": NeuralType(('B', 'T'), LabelsType()), } def __init__( @@ -84,11 +84,11 @@ def output_ports(self): # "input_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "subtokens_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask": NeuralType(ChannelType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "subtokens_mask": NeuralType(ChannelType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_type_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask": NeuralType(('B', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "subtokens_mask": NeuralType(('B', 'T'), ChannelType()), } def __init__( diff --git a/nemo/collections/nlp/nm/losses/aggregator_loss.py b/nemo/collections/nlp/nm/losses/aggregator_loss.py index 8bedfa651790..b1681c7048cb 100644 --- a/nemo/collections/nlp/nm/losses/aggregator_loss.py +++ b/nemo/collections/nlp/nm/losses/aggregator_loss.py @@ -46,7 +46,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, num_inputs=2): # Store number of inputs/losses. diff --git a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py index cbe37ebbec65..ce73176747d7 100644 --- a/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py +++ b/nemo/collections/nlp/nm/losses/joint_intent_slot_loss.py @@ -56,11 +56,11 @@ def input_ports(self): # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "intents": NeuralType({0: AxisType(BatchTag)}), # "slots": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "intent_logits": NeuralType(LogitsType(), ('B', 'D')), - "slot_logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), - "intents": NeuralType(ChannelType(), tuple('B')), - "slots": NeuralType(ChannelType(), ('B', 'T')), + "intent_logits": NeuralType(('B', 'D'), LogitsType()), + "slot_logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), + "intents": NeuralType(tuple('B'), ChannelType()), + "slots": NeuralType(('B', 'T'), ChannelType()), } @property @@ -71,7 +71,7 @@ def output_ports(self): NeuralType(None) """ # return {"loss": NeuralType(None)} - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__( self, num_slots, slot_classes_loss_weights=None, intent_classes_loss_weights=None, intent_loss_weight=0.6, diff --git a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py index 239872d0b17d..38f5169bf348 100644 --- a/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py +++ b/nemo/collections/nlp/nm/losses/masked_language_modeling_loss.py @@ -37,9 +37,9 @@ def input_ports(self): # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "output_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "output_ids": NeuralType(ChannelType(), ('B', 'T')), - "output_mask": NeuralType(ChannelType(), ('B', 'T')), + "logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "output_ids": NeuralType(('B', 'T'), ChannelType()), + "output_mask": NeuralType(('B', 'T'), ChannelType()), } @property @@ -49,7 +49,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, label_smoothing=0.0): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py index 2a4a9c526eca..1564f43c40b0 100644 --- a/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py +++ b/nemo/collections/nlp/nm/losses/padded_smoothed_cross_entropy_loss.py @@ -42,8 +42,8 @@ def input_ports(self): return { # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "target_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "target_ids": NeuralType(LabelsType(), ('B', 'T')), + "logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "target_ids": NeuralType(('B', 'T'), LabelsType()), } @property @@ -51,7 +51,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"loss": NeuralType(None)} - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, pad_id, label_smoothing=0, predict_last_k=0): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/losses/qa_squad_loss.py b/nemo/collections/nlp/nm/losses/qa_squad_loss.py index 9f7fe2461232..1237b9255edb 100644 --- a/nemo/collections/nlp/nm/losses/qa_squad_loss.py +++ b/nemo/collections/nlp/nm/losses/qa_squad_loss.py @@ -43,9 +43,9 @@ def input_ports(self): # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "start_positions": NeuralType({0: AxisType(BatchTag)}), # "end_positions": NeuralType({0: AxisType(BatchTag)}), - "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "start_positions": NeuralType(ChannelType(), tuple('B')), - "end_positions": NeuralType(ChannelType(), tuple('B')), + "logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "start_positions": NeuralType(tuple('B'), ChannelType()), + "end_positions": NeuralType(tuple('B'), ChannelType()), } @property @@ -69,9 +69,9 @@ def output_ports(self): # "loss": NeuralType(None), # "start_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "end_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "loss": NeuralType(LossType()), - "start_logits": NeuralType(ChannelType(), ('B', 'T')), - "end_logits": NeuralType(ChannelType(), ('B', 'T')), + "loss": NeuralType(elements_type=LossType()), + "start_logits": NeuralType(('B', 'T'), ChannelType()), + "end_logits": NeuralType(('B', 'T'), ChannelType()), } def __init__(self): diff --git a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py index cf01cbfe3e33..ea065494e8ee 100644 --- a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py +++ b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py @@ -73,9 +73,9 @@ def input_ports(self): # ), # "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "logits": NeuralType(LogitsType(), ('B', 'T', 'D', 'D')), - "targets": NeuralType(ChannelType(), ('B', 'D', 'T')), - "loss_mask": NeuralType(LengthsType(), ('B', 'D')), + "logits": NeuralType(('B', 'T', 'D', 'D'), LogitsType()), + "targets": NeuralType(('B', 'D', 'T'), ChannelType()), + "loss_mask": NeuralType(('B', 'D'), LengthsType()), } @property @@ -83,7 +83,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"loss": NeuralType(None)} - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self): LossNM.__init__(self) @@ -126,8 +126,8 @@ def input_ports(self): return { # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), - "logits": NeuralType(LogitsType(), ('B', 'D', 'D')), - "labels": NeuralType(LabelsType(), ('B', 'D')), + "logits": NeuralType(('B', 'D', 'D'), LogitsType()), + "labels": NeuralType(('B', 'D'), LabelsType()), } @property @@ -135,7 +135,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"loss": NeuralType(None)} - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, num_classes, **kwargs): LossNM.__init__(self, **kwargs) diff --git a/nemo/collections/nlp/nm/losses/token_classification_loss.py b/nemo/collections/nlp/nm/losses/token_classification_loss.py index 46a651ebe2b2..e27c74e952a3 100644 --- a/nemo/collections/nlp/nm/losses/token_classification_loss.py +++ b/nemo/collections/nlp/nm/losses/token_classification_loss.py @@ -43,9 +43,9 @@ def input_ports(self): # "logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "labels": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "logits": NeuralType(LogitsType(), ('B', 'T', 'D')), - "labels": NeuralType(LabelsType(), ('B', 'T')), - "loss_mask": NeuralType(ChannelType(), ('B', 'T')), + "logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "labels": NeuralType(('B', 'T'), LabelsType()), + "loss_mask": NeuralType(('B', 'T'), ChannelType()), } @property @@ -55,7 +55,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, num_classes, class_weights=None): LossNM.__init__(self) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py index 252701ab92b5..e51ca6b3b9d1 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/bert_nm.py @@ -54,9 +54,9 @@ def input_ports(self): # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "token_type_ids": NeuralType(ChannelType(), ('B', 'T')), - "attention_mask": NeuralType(ChannelType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "token_type_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), ChannelType()), } @property @@ -64,7 +64,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py index 90084947876c..60b1f2c45e7c 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_classification_nm.py @@ -42,14 +42,14 @@ class SequenceClassifier(TrainableNM): def input_ports(self): """Returns definitions of module input ports. """ - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} - return {"logits": NeuralType(LogitsType(), ('B', 'D'))} + return {"logits": NeuralType(('B', 'D'), LogitsType())} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py index d3aed9955da2..0989afd162ad 100644 --- a/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/sequence_regression_nm.py @@ -41,14 +41,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"preds": NeuralType({0: AxisType(RegressionTag)})} - return {"preds": NeuralType(RegressionValuesType(), tuple('B'))} + return {"preds": NeuralType(tuple('B'), RegressionValuesType())} def __init__(self, hidden_size, num_layers=2, activation='relu', dropout=0.0, use_transformer_pretrained=True): super().__init__() diff --git a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py index ff5de50d620e..1b4c879906c7 100644 --- a/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/token_classification_nm.py @@ -44,14 +44,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"logits": NeuralType(LogitsType(), ('B', 'T', 'C'))} + return {"logits": NeuralType(('B', 'T', 'C'), LogitsType())} def __init__( self, @@ -105,14 +105,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'C'))} + return {"hidden_states": NeuralType(('B', 'T', 'C'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"logits": NeuralType(LogitsType(), ('B', 'T', 'D'))} + return {"logits": NeuralType(('B', 'T', 'D'), LogitsType())} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py index 0822d769d246..db858982adb1 100644 --- a/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/transformer/transformer_nm.py @@ -51,8 +51,8 @@ def input_ports(self): return { # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids": NeuralType(ChannelType(), ('B', 'T')), - "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "input_mask_src": NeuralType(('B', 'T'), ChannelType()), } @property @@ -61,7 +61,7 @@ def output_ports(self): """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( self, @@ -142,10 +142,10 @@ def input_ports(self): # "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "input_mask_tgt": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "input_ids_tgt": NeuralType(ChannelType(), ('B', 'T')), - "hidden_states_src": NeuralType(ChannelType(), ('B', 'T', 'D')), - "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), - "input_mask_tgt": NeuralType(ChannelType(), ('B', 'T')), + "input_ids_tgt": NeuralType(('B', 'T'), ChannelType()), + "hidden_states_src": NeuralType(('B', 'T', 'D'), ChannelType()), + "input_mask_src": NeuralType(('B', 'T'), ChannelType()), + "input_mask_tgt": NeuralType(('B', 'T'), ChannelType()), } @property @@ -153,7 +153,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'D'))} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( self, @@ -220,14 +220,14 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"input_ids": NeuralType(ChannelType(), ('B', 'T'))} + return {"input_ids": NeuralType(('B', 'T'), ChannelType())} @property def output_ports(self): """Returns definitions of module output ports. """ # return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"output_ids": NeuralType(ChannelType(), ('B', 'T'))} + return {"output_ids": NeuralType(('B', 'T'), ChannelType())} def __init__(self, decoder, log_softmax, max_seq_length, pad_token, bos_token, eos_token, batch_size=1): super().__init__() @@ -278,8 +278,8 @@ def input_ports(self): return { # "hidden_states_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), # "input_mask_src": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "hidden_states_src": NeuralType(ChannelType(), ('B', 'T', 'C')), - "input_mask_src": NeuralType(ChannelType(), ('B', 'T')), + "hidden_states_src": NeuralType(('B', 'T', 'C'), ChannelType()), + "input_mask_src": NeuralType(('B', 'T'), ChannelType()), } @property @@ -287,7 +287,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"output_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"output_ids": NeuralType(ChannelType(), ('B', 'T'))} + return {"output_ids": NeuralType(('B', 'T'), ChannelType())} @property def num_weights(self): diff --git a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py index bd29209918c2..1e047542e3ba 100644 --- a/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py +++ b/nemo/collections/nlp/nm/trainables/dialogue_state_tracking/state_tracking_trade_nm.py @@ -72,12 +72,12 @@ def input_ports(self): # 'input_lens': NeuralType({0: AxisType(BatchTag)}), # 'src_ids': NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # 'targets': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), - 'encoder_hidden': NeuralType(ChannelType(), ('B', 'T', 'C')), - 'encoder_outputs': NeuralType(ChannelType(), ('B', 'T', 'C')), - 'input_lens': NeuralType(LengthsType(), tuple('B')), - 'src_ids': NeuralType(ChannelType(), ('B', 'T')), + 'encoder_hidden': NeuralType(('B', 'T', 'C'), ChannelType()), + 'encoder_outputs': NeuralType(('B', 'T', 'C'), ChannelType()), + 'input_lens': NeuralType(tuple('B'), LengthsType()), + 'src_ids': NeuralType(('B', 'T'), ChannelType()), # 'targets': NeuralType(ChannelType(), ('B', 'D', 'T')), - 'targets': NeuralType(LabelsType(), ('B', 'D', 'T')), + 'targets': NeuralType(('B', 'D', 'T'), LabelsType()), } @property @@ -96,8 +96,8 @@ def output_ports(self): # 'gate_outputs': NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(ChannelTag)}), # } return { - 'point_outputs': NeuralType(LogitsType(), ('B', 'T', 'D', 'D')), - 'gate_outputs': NeuralType(LogitsType(), ('B', 'D', 'D')), + 'point_outputs': NeuralType(('B', 'T', 'D', 'D'), LogitsType()), + 'gate_outputs': NeuralType(('B', 'D', 'D'), LogitsType()), } def __init__(self, vocab, embeddings, hid_size, dropout, slots, nb_gate, teacher_forcing=0.5): diff --git a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py index b751df91df28..c906417afd6d 100644 --- a/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py +++ b/nemo/collections/nlp/nm/trainables/joint_intent_slot/joint_intent_slot_nm.py @@ -41,7 +41,7 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} - return {"hidden_states": NeuralType(ChannelType(), ('B', 'T', 'C'))} + return {"hidden_states": NeuralType(('B', 'T', 'C'), ChannelType())} @property def output_ports(self): @@ -62,8 +62,8 @@ def output_ports(self): return { # "intent_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), # "slot_logits": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)}), - "intent_logits": NeuralType(LogitsType(), ('B', 'D')), - "slot_logits": NeuralType(LogitsType(), ('B', 'T', 'D')), + "intent_logits": NeuralType(('B', 'D'), LogitsType()), + "slot_logits": NeuralType(('B', 'T', 'D'), LogitsType()), } def __init__(self, hidden_size, num_intents, num_slots, dropout=0.0, use_transformer_pretrained=True, **kwargs): diff --git a/nemo/collections/simple_gan/gan.py b/nemo/collections/simple_gan/gan.py index dd2028ba769d..b0d39a406d64 100644 --- a/nemo/collections/simple_gan/gan.py +++ b/nemo/collections/simple_gan/gan.py @@ -27,7 +27,7 @@ def input_ports(self): # 3: AxisType(WidthTag, 28), # } # ) - "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) + "image": NeuralType(('B', 'C', 'H', 'W'), ChannelType()) } @property @@ -35,7 +35,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)})} - return {"decision": NeuralType(ChannelType(), ('B', 'C'))} + return {"decision": NeuralType(('B', 'C'), ChannelType())} def __init__(self): super().__init__() @@ -77,7 +77,7 @@ def input_ports(self): # 3: AxisType(WidthTag, 4), # } # ) - "latents": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) + "latents": NeuralType(('B', 'C', 'H', 'W'), ChannelType()) } @property @@ -93,7 +93,7 @@ def output_ports(self): # 3: AxisType(WidthTag, 28), # } # ) - "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) + "image": NeuralType(('B', 'C', 'H', 'W'), ChannelType()) } def __init__(self, batch_size): @@ -138,14 +138,14 @@ def input_ports(self): """ return { # "decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), - "decision": NeuralType(ChannelType(), ('B', 'D')) + "decision": NeuralType(('B', 'D'), ChannelType()) } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, neg=False): super().__init__() @@ -181,8 +181,8 @@ def input_ports(self): # } # ), # "interpolated_decision": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag, 1)}), - "interpolated_image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), - "interpolated_decision": NeuralType(ChannelType(), ('B', 'C')), + "interpolated_image": NeuralType(('B', 'C', 'H', 'W'), ChannelType()), + "interpolated_decision": NeuralType(('B', 'C'), ChannelType()), } @property @@ -192,7 +192,7 @@ def output_ports(self): loss: NeuralType(None) """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, lambda_): super().__init__() @@ -245,8 +245,8 @@ def input_ports(self): # 3: AxisType(WidthTag, 28), # } # ), - "image1": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), - "image2": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), + "image1": NeuralType(('B', 'C', 'H', 'W'), ChannelType()), + "image2": NeuralType(('B', 'C', 'H', 'W'), ChannelType()), } @property @@ -262,7 +262,7 @@ def output_ports(self): # 3: AxisType(WidthTag, 28), # } # ) - "interpolated_image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) + "interpolated_image": NeuralType(('B', 'C', 'H', 'W'), ChannelType()) } def __init__(self): @@ -307,7 +307,7 @@ def output_ports(self): # 3: AxisType(WidthTag, 4), # } # ) - "latent": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')) + "latent": NeuralType(('B', 'C', 'H', 'W'), ChannelType()) } def __init__(self, batch_size): @@ -372,9 +372,9 @@ def output_ports(self): # } # ), # "label": NeuralType({0: AxisType(BatchTag)}), - "latent": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), - "image": NeuralType(ChannelType(), ('B', 'C', 'H', 'W')), - "label": NeuralType(LabelsType(), tuple('B')), + "latent": NeuralType(('B', 'C', 'H', 'W'), ChannelType()), + "image": NeuralType(('B', 'C', 'H', 'W'), ChannelType()), + "label": NeuralType(tuple('B'), LabelsType()), } def __init__(self, batch_size, root, train=True, shuffle=True): diff --git a/nemo/collections/tts/data_layers.py b/nemo/collections/tts/data_layers.py index 89344ec85583..ffebe99e3df9 100644 --- a/nemo/collections/tts/data_layers.py +++ b/nemo/collections/tts/data_layers.py @@ -52,8 +52,8 @@ def output_ports(self): return { # "audio_signal": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "a_sig_length": NeuralType({0: AxisType(BatchTag)}), - "audio_signal": NeuralType(AudioSignal(), ('B', 'T')), - "a_sig_length": NeuralType(LengthsType(), tuple('B')), + "audio_signal": NeuralType(('B', 'T'), AudioSignal()), + "a_sig_length": NeuralType(tuple('B'), LengthsType()), } def __init__( diff --git a/nemo/collections/tts/tacotron2_modules.py b/nemo/collections/tts/tacotron2_modules.py index 9399bfa85d53..083ac4697526 100644 --- a/nemo/collections/tts/tacotron2_modules.py +++ b/nemo/collections/tts/tacotron2_modules.py @@ -37,7 +37,7 @@ def input_ports(self): """Returns definitions of module input ports. """ # return {"char_phone": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"char_phone": NeuralType(LabelsType(), ('B', 'T'))} + return {"char_phone": NeuralType(('B', 'T'), LabelsType())} @property def output_ports(self): @@ -47,7 +47,7 @@ def output_ports(self): # "char_phone_embeddings": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} # ) - "char_phone_embeddings": NeuralType(EmbeddedTextType(), ('B', 'D', 'T')) + "char_phone_embeddings": NeuralType(('B', 'D', 'T'), EmbeddedTextType()) } def __init__(self, n_symbols, symbols_embedding_dim: int = 512): @@ -83,8 +83,8 @@ def input_ports(self): # {0: AxisType(BatchTag), 1: AxisType(EmbeddedTextTag), 2: AxisType(TimeTag),} # ), # "embedding_length": NeuralType({0: AxisType(BatchTag)}), - "char_phone_embeddings": NeuralType(EmbeddedTextType(), ('B', 'D', 'T')), - "embedding_length": NeuralType(LengthsType(), tuple('B')), + "char_phone_embeddings": NeuralType(('B', 'D', 'T'), EmbeddedTextType()), + "embedding_length": NeuralType(tuple('B'), LengthsType()), } @property @@ -95,7 +95,7 @@ def output_ports(self): # "char_phone_encoded": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} # ) - "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')) + "char_phone_encoded": NeuralType(('B', 'T', 'D'), EncodedRepresentation()) } def __init__( @@ -164,9 +164,9 @@ def input_ports(self): # "mel_target": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ), - "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), - "encoded_length": NeuralType(LengthsType(), tuple('B')), - "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "char_phone_encoded": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "encoded_length": NeuralType(tuple('B'), LengthsType()), + "mel_target": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), } @property @@ -179,9 +179,9 @@ def output_ports(self): # ), # "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), - "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "gate_output": NeuralType(ChannelType(), ('B', 'T')), - "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')), + "mel_output": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_output": NeuralType(('B', 'T'), ChannelType()), + "alignments": NeuralType(('B', 'T', 'T'), ChannelType()), } def __init__( @@ -278,8 +278,8 @@ def input_ports(self): # {0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(EncodedRepresentationTag),} # ), # "encoded_length": NeuralType({0: AxisType(BatchTag)}), - "char_phone_encoded": NeuralType(EncodedRepresentation(), ('B', 'T', 'D')), - "encoded_length": NeuralType(LengthsType(), tuple('B')), + "char_phone_encoded": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "encoded_length": NeuralType(tuple('B'), LengthsType()), } @property @@ -293,10 +293,10 @@ def output_ports(self): # "gate_output": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "alignments": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(TimeTag),}), # "mel_len": NeuralType({0: AxisType(BatchTag)}), - "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "gate_output": NeuralType(ChannelType(), ('B', 'T')), - "alignments": NeuralType(ChannelType(), ('B', 'T', 'T')), - "mel_len": NeuralType(LengthsType(), tuple('B')), + "mel_output": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_output": NeuralType(('B', 'T'), ChannelType()), + "alignments": NeuralType(('B', 'T', 'T'), ChannelType()), + "mel_len": NeuralType(tuple('B'), LengthsType()), } def __str__(self): @@ -336,7 +336,7 @@ def input_ports(self): # "mel_input": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ) - "mel_input": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) + "mel_input": NeuralType(('B', 'D', 'T'), MelSpectrogramType()) } @property @@ -347,7 +347,7 @@ def output_ports(self): # "mel_output": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ), - "mel_output": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) + "mel_output": NeuralType(('B', 'D', 'T'), MelSpectrogramType()) } def __init__( @@ -405,20 +405,20 @@ def input_ports(self): # "gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "target_len": NeuralType({0: AxisType(BatchTag)}), # "seq_len": NeuralType({0: AxisType(BatchTag)}), - "mel_out": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "mel_out_postnet": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "gate_out": NeuralType(ChannelType(), ('B', 'T')), - "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "gate_target": NeuralType(ChannelType(), ('B', 'T')), - "target_len": NeuralType(LengthsType(), tuple('B')), - "seq_len": NeuralType(LengthsType(), tuple('B')), + "mel_out": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "mel_out_postnet": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_out": NeuralType(('B', 'T'), ChannelType()), + "mel_target": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_target": NeuralType(('B', 'T'), ChannelType()), + "target_len": NeuralType(tuple('B'), LengthsType()), + "seq_len": NeuralType(tuple('B'), LengthsType()), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, pad_value: float = -11.52): super().__init__() @@ -476,8 +476,8 @@ def input_ports(self): # "mel_target": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ), - "target_len": NeuralType(LengthsType(), tuple('B')), - "mel_target": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), + "target_len": NeuralType(tuple('B'), LengthsType()), + "mel_target": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), } @property @@ -485,7 +485,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"gate_target": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"gate_target": NeuralType(ChannelType(), ('B', 'T'))} + return {"gate_target": NeuralType(('B', 'T'), ChannelType())} def forward(self, target_len, mel_target): max_len = mel_target.shape[2] diff --git a/nemo/collections/tts/waveglow_modules.py b/nemo/collections/tts/waveglow_modules.py index 06439d272ff2..1acffdb59d73 100644 --- a/nemo/collections/tts/waveglow_modules.py +++ b/nemo/collections/tts/waveglow_modules.py @@ -47,8 +47,8 @@ def input_ports(self): # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ), # "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "mel_spectrogram": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')), - "audio": NeuralType(AudioSignal(), ('B', 'T')), + "mel_spectrogram": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "audio": NeuralType(('B', 'T'), AudioSignal()), } @property @@ -60,9 +60,9 @@ def output_ports(self): # "audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "log_s_list": NeuralType(), # "log_det_W_list": NeuralType(), - "audio": NeuralType(AudioSignal(), ('B', 'T')), - "log_s_list": NeuralType(ChannelType()), - "log_det_W_list": NeuralType(ChannelType()), + "audio": NeuralType(('B', 'T'), AudioSignal()), + "log_s_list": NeuralType(elements_type=ChannelType()), + "log_det_W_list": NeuralType(elements_type=ChannelType()), } def __init__( @@ -143,7 +143,7 @@ def input_ports(self): # "mel_spectrogram": NeuralType( # {0: AxisType(BatchTag), 1: AxisType(MelSpectrogramSignalTag), 2: AxisType(TimeTag),} # ) - "mel_spectrogram": NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) + "mel_spectrogram": NeuralType(('B', 'D', 'T'), MelSpectrogramType()) } @property @@ -151,7 +151,7 @@ def output_ports(self): """Returns definitions of module output ports. """ # return {"audio": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)})} - return {"audio": NeuralType(AudioSignal(), ('B', 'T'))} + return {"audio": NeuralType(('B', 'T'), AudioSignal())} def __str__(self): return "WaveGlowNM" @@ -233,16 +233,16 @@ def input_ports(self): # "z": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), # "log_s_list": NeuralType(), # "log_det_W_list": NeuralType(), - "z": NeuralType(AudioSignal(), ('B', 'T')), - "log_s_list": NeuralType(ChannelType()), - "log_det_W_list": NeuralType(ChannelType()), + "z": NeuralType(('B', 'T'), AudioSignal()), + "log_s_list": NeuralType(elements_type=ChannelType()), + "log_det_W_list": NeuralType(elements_type=ChannelType()), } @property def output_ports(self): """Returns definitions of module output ports. """ - return {"loss": NeuralType(LossType())} + return {"loss": NeuralType(elements_type=LossType())} def __init__(self, sigma: float = 1.0): super().__init__() diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 0692ea46095c..7a7a5154ef2c 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -463,14 +463,12 @@ def __get_pytorch_module(self, name, collection, params, pretrained): _nm_name = name.lower() if _nm_name == "resnet18": input_ports = { - "x": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - } - ) + "x": NeuralType({ + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, 224), + 3: AxisType(WidthTag, 224), + }) } output_ports = {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} @@ -483,14 +481,12 @@ def __get_pytorch_module(self, name, collection, params, pretrained): ) elif _nm_name == "resnet50": input_ports = { - "x": NeuralType( - { - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - } - ) + "x": NeuralType({ + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, 224), + 3: AxisType(WidthTag, 224), + }) } output_ports = {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 25e42c7824fa..74bf9cb6108d 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -311,6 +311,7 @@ def __call__(self, **kwargs): return result + def __str__(self): return self.__class__.__name__ diff --git a/nemo/core/neural_types/axes.py b/nemo/core/neural_types/axes.py index acb9a27646f2..dcc2e7736ff6 100644 --- a/nemo/core/neural_types/axes.py +++ b/nemo/core/neural_types/axes.py @@ -32,11 +32,15 @@ class AxisKindAbstract(Enum): class AxisKind(AxisKindAbstract): """This Enum represents what does varying axis dimension mean. - For example, does this dimension correspond to width, batch, time, etc.""" + For example, does this dimension correspond to width, batch, time, etc. + The "Dimension" and "Channel" kinds are the same and used to represent + a general axis. + """ Batch = 0 Time = 1 Dimension = 2 + Channel = 2 Width = 3 Height = 4 @@ -64,9 +68,10 @@ def from_str(label): class AxisType(object): """This class represents axis semantics and (optionally) it's dimensionality Args: - kind (AxisKindAbstract): - size (int, optional): - is_list (bool, default=False): + kind (AxisKindAbstract): what kind of axis it is? For example Batch, Height, etc. + size (int, optional): specify if the axis should have a fixed size. By default it is set to None and you + typically do not want to set it for Batch and Time + is_list (bool, default=False): whether this is a list or a tensor axis """ def __init__(self, kind: AxisKindAbstract, size: Optional[int] = None, is_list=False): diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index dd7fcd754f98..59f818ee2688 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -44,19 +44,25 @@ class ElementType(ABC): """Abstract class defining semantics of the tensor elements. - We are replying on Python for inheritance checking""" + We are relying on Python for inheritance checking""" - @abstractmethod - def __str__(cls): - pass + def __str__(self): + self.__doc__ @property def type_parameters(self) -> Dict: - """Override this property to parametrize your type""" + """Override this property to parametrize your type. For example, you can specify 'storage' type such as + float, int, bool with 'dtype' keyword. Another example, is if you want to represent a signal with a + particular property (say, sample frequency), then you can put sample_freq->value in there. + When two types are compared their type_parameters must match.""" return {} @property def fields(self) -> Optional[Tuple]: + """This should be used to logically represent tuples/structures. For example, if you want to represent a + bounding box (x, y, width, height) you can put a tuple with names ('x', y', 'w', 'h') in here. + Under the hood this should be converted to the last tesnor dimension of fixed size = len(fields). + When two types are compared their fields must match.""" return None def compare(self, second) -> NeuralTypeComparisonResult: @@ -92,67 +98,61 @@ def compare(self, second) -> NeuralTypeComparisonResult: class VoidType(ElementType): - """Void-like type which is compatible with everything + """Void-like type which is compatible with everything. + It is a good practice to use this type only as necessary. + For example, when you need template-like functionality. """ - - def __str__(self): - return str("void type. compatible with everything") - def compare(cls, second: abc.ABCMeta) -> NeuralTypeComparisonResult: return NeuralTypeComparisonResult.SAME # TODO: Consider moving these files elsewhere class ChannelType(ElementType): - def __str__(self): - return "convolutional channel value" + """Element to represent convolutional input/output channel. + """ class EmbeddedTextType(ChannelType): - def __str__(self): - return "text embedding" + """Element to represent output on word/text embedding layers + """ class LogitsType(ElementType): - def __str__(self): - return "neural type representing logits" + """Element type to represent logits""" class LogprobsType(ElementType): - def __str__(self): - return "neural type representing log probabilities" + """Element type to represent log-probabilities. For example, outputs of softmax layers.""" class LabelsType(ElementType): - def __str__(self): - return "neural type representing labels" + """Element type to represent some sort of labels. This is often used as a base class to create + a more concrete types such as RegressionValuesType, etc.""" class LengthsType(ElementType): - def __str__(self): - return "neural type representing lengths of something" + """Element type representing lengths of something""" class LossType(ElementType): - def __str__(self): - return "neural type representing loss value" + """Element type to represent outputs of Loss modules""" class EncodedRepresentation(ChannelType): - def __str__(self): - return "encoded representation, for example, encoder's output" + """Element type to represent encoded representation, for example, encoder's output""" class AcousticEncodedRepresentation(EncodedRepresentation): - def __str__(self): - return "encoded representation returned by the acoustic encoder model" + """Element type to represent encoded representation returned by the acoustic encoder model""" class AudioSignal(ElementType): - def __str__(self): - return "encoded representation returned by the acoustic encoder model" - - def __init__(self, freq=16000): + """Element type to represent encoded representation returned by the acoustic encoder model + Args: + freq (int): sampling frequency of a signal. Note that two signals will only be the same if their + freq is the same. + """ + def __init__(self, freq: int = 16000): self._params = {} self._params['freq'] = freq @@ -162,30 +162,24 @@ def type_parameters(self): class SpectrogramType(ChannelType): - def __str__(self): - return "generic spectorgram type" + """Element type to represent generic spectrogram signal""" class MelSpectrogramType(SpectrogramType): - def __str__(self): - return "mel spectorgram type" + """Element type to represent mel spectrogram signal""" class MFCCSpectrogramType(SpectrogramType): - def __str__(self): - return "mfcc spectorgram type" + """Element type to represent MFCC spectrogram signal""" class PredictionsType(LabelsType): - def __str__(self): - return "predictions values type" + """Element type to represent some sort of predictions returned by model""" class RegressionValuesType(PredictionsType): - def __str__(self): - return "regression values type" + """Element type to represent labels for regression task""" class CategoricalValuesType(PredictionsType): - def __str__(self): - return "regression values type" + """Element type to represent labels for categorical classification task""" diff --git a/nemo/core/neural_types/neural_type.py b/nemo/core/neural_types/neural_type.py index b0c1a310ec33..a2070c354b3c 100644 --- a/nemo/core/neural_types/neural_type.py +++ b/nemo/core/neural_types/neural_type.py @@ -27,16 +27,26 @@ import uuid from typing import Optional, Tuple -from .axes import AxisKind, AxisType -from .comparison import NeuralTypeComparisonResult -from .elements import * +from nemo.core.neural_types.axes import AxisKind, AxisType +from nemo.core.neural_types.comparison import NeuralTypeComparisonResult +from nemo.core.neural_types.elements import * class NeuralType(object): """This is the main class which would represent neural type concept. - nmTensors derives from this. It is used to represent *the types* of inputs and outputs.""" - - def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple] = None, optional=False): + nmTensors derives from this. It is used to represent *the types* of inputs and outputs. + Args: + axes (Optional[Tuple]): a tuple of AxisTypes objects representing the semantics of what varying each axis means + You can use a short, string-based form here. For example: ('B', 'C', 'H', 'W') would correspond to an NCHW + format frequently used in computer vision. ('B', 'T', 'D') is frequently used for signal processing and + means [batch, time, dimension/channel]. + elements_type (ElementType): an instance of ElementType class representing the semantics of what is stored + inside the tensor. For example: logits (LogitsType), log probabilities (LogprobType), etc. + optional (bool): By default, this is false. If set to True, it would means that input to the port of this + type can be optional. + """ + + def __init__(self, axes: Optional[Tuple] = None, elements_type: ElementType = VoidType(), optional=False): if not isinstance(elements_type, ElementType): raise ValueError( f"elements_type of NeuralType must be an instance of a class derived from ElementType." @@ -59,6 +69,8 @@ def __init__(self, elements_type: ElementType = VoidType(), axes: Optional[Tuple self.optional = optional def compare(self, second) -> NeuralTypeComparisonResult: + """Performs neural type comparison of self with second. When you chain two modules' inputs/outputs via + __call__ method, this comparison will be called to ensure neural type compatibility.""" # First, handle dimensionality axes_a = self.axes axes_b = second.axes @@ -180,7 +192,7 @@ def __init__(self, producer, producer_args, name, ntype=None): producer_args (dict): a dictionary of port_name->NmTensor value of arguments which were sent to producer to create this """ - super(NmTensor, self).__init__(elements_type=ntype.elements_type, axes=ntype.axes, optional=ntype.optional) + super(NmTensor, self).__init__(axes=ntype.axes, elements_type=ntype.elements_type, optional=ntype.optional) self._producer = producer self._producer_args = producer_args self._name = name diff --git a/tests/asr/test_zeroDS.py b/tests/asr/test_zeroDS.py index e2c9bd6f7373..6dc9926a597d 100644 --- a/tests/asr/test_zeroDS.py +++ b/tests/asr/test_zeroDS.py @@ -108,12 +108,11 @@ def test_asr_with_zero_ds(self): # "transcript": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag, dim=64)}), # "transcript_length": NeuralType({0: AxisType(BatchTag)}), "processed_signal": NeuralType( - SpectrogramType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 64), AxisType(AxisKind.Time, 64)), - ), - "processed_length": NeuralType(LengthsType(), tuple('B')), - "transcript": NeuralType(LabelsType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64))), - "transcript_length": NeuralType(LengthsType(), tuple('B')), + SpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), + "transcript": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64)), LabelsType()), + "transcript_length": NeuralType(tuple('B'), LengthsType()), }, ) diff --git a/tests/core/test_infer.py b/tests/core/test_infer.py index 811da4560a63..d9b11a3997da 100644 --- a/tests/core/test_infer.py +++ b/tests/core/test_infer.py @@ -31,12 +31,12 @@ def __init__(self): @property def input_ports(self): # return {"mod_in": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} - return {"mod_in": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} + return {"mod_in": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType())} @property def output_ports(self): # return {"mod_out": NeuralType({0: AxisType(BatchTag), 1: AxisType(BaseTag, dim=1)})} - return {"mod_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} + return {"mod_out": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType())} def forward(self, mod_in): return mod_in + 10 @@ -48,11 +48,11 @@ def __init__(self): @property def input_ports(self): - return {"mod_in": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} + return {"mod_in": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType())} @property def output_ports(self): - return {"mod_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)))} + return {"mod_out": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType())} def forward(self, mod_in): return mod_in - 10 @@ -69,7 +69,7 @@ def test_infer_caching(self): dtype=torch.FloatTensor, batch_size=1, output_ports={ - "dl_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1))) + "dl_out": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType()) }, ) addten = AddsTen() @@ -98,7 +98,7 @@ def test_infer_errors(self): dtype=torch.FloatTensor, batch_size=1, output_ports={ - "dl_out": NeuralType(ChannelType(), (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1))) + "dl_out": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 1)), ChannelType()) }, ) addten = AddsTen() diff --git a/tests/core/test_neural_modules.py b/tests/core/test_neural_modules.py index 92dd80237d91..5484285b8e50 100644 --- a/tests/core/test_neural_modules.py +++ b/tests/core/test_neural_modules.py @@ -23,9 +23,8 @@ class NeuralModulesTests(NeMoUnitTest): def test_call_TaylorNet(self): - x_tg = nemo.core.neural_modules.NmTensor( - producer=None, producer_args=None, name=None, ntype=NeuralType(ChannelType(), ('B', 'D')) - ) + x_tg = nemo.core.neural_modules.NmTensor(producer=None, producer_args=None, name=None, ntype=NeuralType(( + 'B', 'D'), ChannelType())) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) # note that real port's name: x was used diff --git a/tests/core/test_neural_modules_pytorch.py b/tests/core/test_neural_modules_pytorch.py index f6d1ce30953b..8f43f2d7356f 100644 --- a/tests/core/test_neural_modules_pytorch.py +++ b/tests/core/test_neural_modules_pytorch.py @@ -69,9 +69,8 @@ def test_constructor_TaylorNet(self): self.assertEqual(tn.init_params["dim"], 4) def test_call_TaylorNet(self): - x_tg = nemo.core.neural_modules.NmTensor( - producer=None, producer_args=None, name=None, ntype=NeuralType(ChannelType(), ('B', 'D')) - ) + x_tg = nemo.core.neural_modules.NmTensor(producer=None, producer_args=None, name=None, ntype=NeuralType( + elements_type=ChannelType(), axes=('B', 'D'))) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) # note that real port's name: x was used diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index a860c889bc9f..c82740c6a712 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -36,17 +36,16 @@ class NeuralTypeSystemTests(NeMoUnitTest): def test_short_vs_long_version(self): long_version = NeuralType( - elements_type=AcousticEncodedRepresentation(), axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)), - ) - short_version = NeuralType(AcousticEncodedRepresentation(), ('B', 'D', 'T')) + elements_type=AcousticEncodedRepresentation()) + short_version = NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()) self.assertEqual(long_version.compare(short_version), NeuralTypeComparisonResult.SAME) self.assertEqual(short_version.compare(long_version), NeuralTypeComparisonResult.SAME) def test_parameterized_type_audio_sampling_frequency(self): - audio16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) - audio8K = NeuralType(AudioSignal(8000), axes=('B', 'T')) - another16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) + audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + audio8K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(8000)) + another16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) self.assertEqual(audio8K.compare(audio16K), NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS) self.assertEqual(audio16K.compare(audio8K), NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS) @@ -60,14 +59,14 @@ def test_transpose_same_1(self): self.assertEqual(type2.compare(type1), NeuralTypeComparisonResult.TRANSPOSE_SAME) def test_transpose_same_2(self): - audio16K = NeuralType(AudioSignal(16000), axes=('B', 'T')) - audio16K_t = NeuralType(AudioSignal(16000), axes=('T', 'B')) + audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + audio16K_t = NeuralType(axes=('T', 'B'), elements_type=AudioSignal(16000)) self.assertEqual(audio16K.compare(audio16K_t), NeuralTypeComparisonResult.TRANSPOSE_SAME) def test_inheritance_spec_augment_example(self): - input = NeuralType(SpectrogramType(), ('B', 'D', 'T')) - out1 = NeuralType(MelSpectrogramType(), ('B', 'D', 'T')) - out2 = NeuralType(MFCCSpectrogramType(), ('B', 'D', 'T')) + input = NeuralType(('B', 'D', 'T'), SpectrogramType()) + out1 = NeuralType(('B', 'D', 'T'), MelSpectrogramType()) + out2 = NeuralType(('B', 'D', 'T'), MFCCSpectrogramType()) self.assertEqual(out1.compare(out2), NeuralTypeComparisonResult.INCOMPATIBLE) self.assertEqual(out2.compare(out1), NeuralTypeComparisonResult.INCOMPATIBLE) self.assertEqual(input.compare(out1), NeuralTypeComparisonResult.GREATER) @@ -82,63 +81,51 @@ def test_singletone(self): self.assertEqual(loss_output2.compare(loss_output1), NeuralTypeComparisonResult.SAME) def test_list_of_lists(self): - T1 = NeuralType( - elements_type=ChannelType(), - axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=True), - AxisType(kind=AxisKind.Time, size=None, is_list=True), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), - ) - T2 = NeuralType( - elements_type=ChannelType(), - axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=False), - AxisType(kind=AxisKind.Time, size=None, is_list=False), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), - ) + T1 = NeuralType(axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), elements_type=ChannelType()) + T2 = NeuralType(axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), elements_type=ChannelType()) # TODO: should this be incompatible instead??? self.assertEqual(T1.compare(T2), NeuralTypeComparisonResult.TRANSPOSE_SAME) def test_void(self): - btc_spctr = NeuralType(SpectrogramType(), ('B', 'T', 'C')) - btc_spct_bad = NeuralType(SpectrogramType(), ('B', 'T')) - btc_void = NeuralType(VoidType(), ('B', 'T', 'C')) + btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) + btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) + btc_void = NeuralType(('B', 'T', 'C'), VoidType()) self.assertEqual(btc_void.compare(btc_spctr), NeuralTypeComparisonResult.SAME) self.assertEqual(btc_spctr.compare(btc_void), NeuralTypeComparisonResult.INCOMPATIBLE) self.assertEqual(btc_void.compare(btc_spct_bad), NeuralTypeComparisonResult.INCOMPATIBLE) def test_big_void(self): - big_void_1 = NeuralType(VoidType()) + big_void_1 = NeuralType(elements_type=VoidType()) big_void_2 = NeuralType() - btc_spctr = NeuralType(SpectrogramType(), ('B', 'T', 'C')) - btc_spct_bad = NeuralType(SpectrogramType(), ('B', 'T')) - t1 = NeuralType( - elements_type=ChannelType(), - axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=True), - AxisType(kind=AxisKind.Time, size=None, is_list=True), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), - ) - t2 = NeuralType( - elements_type=ChannelType(), - axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=False), - AxisType(kind=AxisKind.Time, size=None, is_list=False), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), - ) + btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) + btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) + t1 = NeuralType(axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), elements_type=ChannelType()) + t2 = NeuralType(axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), elements_type=ChannelType()) self.assertEqual(big_void_1.compare(btc_spctr), NeuralTypeComparisonResult.SAME) self.assertEqual(big_void_1.compare(btc_spct_bad), NeuralTypeComparisonResult.SAME) @@ -169,10 +156,8 @@ def wrong(): self.assertRaises(NeuralPortNmTensorMismatchError, wrong) def test_unspecified_dimensions(self): - t0 = NeuralType( - SpectrogramType(), - (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), - ) - t1 = NeuralType(SpectrogramType(), ('B', 'T', 'C')) + t0 = NeuralType((AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), + SpectrogramType()) + t1 = NeuralType(('B', 'T', 'C'), SpectrogramType()) self.assertEqual(t1.compare(t0), NeuralTypeComparisonResult.SAME) self.assertEqual(t0.compare(t1), NeuralTypeComparisonResult.DIM_INCOMPATIBLE) From fe1f9d328f5d96dd9283aa358c8f08b2fecddeb4 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 12 Feb 2020 11:55:07 -0800 Subject: [PATCH 48/70] update dataset path Signed-off-by: Jason --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0685082f93b3..70bb719275d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -174,7 +174,7 @@ pipeline { // } stage('GAN O2') { steps { - sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O2 --num_epochs=3 --train_dataset=/home/mrjenkins/TestData/MNIST/' + sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O2 --num_epochs=3 --train_dataset=/home/mrjenkins/TestData/' } } stage('Jasper AN4 O2') { From 1d6730866c6d7ba6575c13ed2066801c41a07377 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 12:14:39 -0800 Subject: [PATCH 49/70] updated nlp documentation for bert, squad, asr_post_processing and prettified module usage Signed-off-by: Yang Zhang --- .../sources/source/nlp/bert_pretraining.rst | 127 +++++++++++------- docs/sources/source/nlp/asr-improvement.rst | 13 +- docs/sources/source/nlp/bert_pretraining.rst | 43 +++--- .../source/nlp/neural_machine_translation.rst | 34 ++--- .../sources/source/nlp/question_answering.rst | 95 ++++++++----- .../asr_postprocessor/asr_postprocessor.py | 7 +- .../BERTPretrainingTutorial.ipynb | 59 +++++--- .../nlp/language_modeling/bert_pretraining.py | 14 +- .../machine_translation_tutorial.py | 2 +- .../question_answering_squad.py | 8 +- 10 files changed, 247 insertions(+), 155 deletions(-) diff --git a/docs/docs_zh/sources/source/nlp/bert_pretraining.rst b/docs/docs_zh/sources/source/nlp/bert_pretraining.rst index e3877298f027..24d3a556020b 100644 --- a/docs/docs_zh/sources/source/nlp/bert_pretraining.rst +++ b/docs/docs_zh/sources/source/nlp/bert_pretraining.rst @@ -5,7 +5,7 @@ BERT预训练 创建一个专门领域的BERT模型对于某些应用是更有优势的。比如一个专门针对生物医学领域的专业BERT，类似于BioBERT :cite:`nlp-bert-lee2019biobert` 和SciBERT :cite:`nlp-bert-beltagy2019scibert` 。 -本教程中所使用的代码来自于 ``examples/nlp/bert_pretraining.py``. +本教程中所使用的代码来自于 ``examples/nlp/language_modeling/bert_pretraining.py``. 语料下载 -------- @@ -51,10 +51,20 @@ BERT预训练 # If you're using a custom vocabulary, create your tokenizer like this tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) + special_tokens = { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + # or + tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") 创建模型 -------- @@ -78,76 +88,99 @@ BERT预训练 .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT( - vocab_size=tokenizer.vocab_size, - num_layers=args.num_layers, - d_model=args.d_model, - num_heads=args.num_heads, - d_inner=args.d_inner, - max_seq_length=args.max_seq_length, - hidden_act="gelu") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT( + vocab_size=args.vocab_size, + num_hidden_layers=args.num_hidden_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + intermediate_size=args.intermediate_size, + max_position_embeddings=args.max_seq_length, + hidden_act=args.hidden_act) 如果你想从一个已有的BERT模型文件继续训练，那设置一个模型的名字即可。如果想查看完整的预训练好的BERT模型列表，可以使用 `nemo_nlp.huggingface.BERT.list_pretrained_models()` 。 .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT(pretrained_model_name="bert-base-cased") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name="bert-base-cased") 接下来，我们需要定义分类器和损失函数。在本教程中，我们会同时使用掩码语言模型和预测下一句模型这两个模型的损失函数，如果你只用掩饰语言模型作为损失的话，可能会观察到更高的准确率。 .. code-block:: python - mlm_classifier = nemo_nlp.TokenClassifier(args.d_model, + mlm_classifier = nemo_nlp.nm.trainables.TokenClassifier(args.d_model, num_classes=tokenizer.vocab_size, num_layers=1, log_softmax=True) - mlm_loss_fn = nemo_nlp.MaskedLanguageModelingLossNM() + mlm_loss_fn = nemo_nlp.nm.losses.MaskedLanguageModelingLossNM() - nsp_classifier = nemo_nlp.SequenceClassifier(args.d_model, + nsp_classifier = nemo_nlp.nm.trainables.SequenceClassifier(args.d_model, num_classes=2, num_layers=2, log_softmax=True) nsp_loss_fn = nemo.backends.pytorch.common.CrossEntropyLoss() - bert_loss = nemo_nlp.LossAggregatorNM(num_inputs=2) + bert_loss = nemo_nlp.nm.losses.LossAggregatorNM(num_inputs=2) 然后，我们把从输入到输出的整个计算流程封装成一个函数。有了这个函数，我们就可以很方便的分别创建训练流和评估流： .. code-block:: python def create_pipeline(**args): - dataset = nemo_nlp.BertPretrainingDataset(**params) - data_layer = nemo_nlp.BertPretrainingDataLayer(dataset) - steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus) - - input_ids, input_type_ids, input_mask, \ - output_ids, output_mask, nsp_labels = data_layer() - - hidden_states = bert_model(input_ids=input_ids, - token_type_ids=input_type_ids, - attention_mask=input_mask) - - mlm_logits = mlm_classifier(hidden_states=hidden_states) - mlm_loss = mlm_loss_fn(logits=mlm_logits, - output_ids=output_ids, - output_mask=output_mask) - - nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) - - loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) - - return loss, [mlm_loss, nsp_loss], steps_per_epoch - - - train_loss, _, steps_per_epoch = create_pipeline(data_desc.train_file, - args.max_seq_length, - args.mask_probability, - args.batch_size) - eval_loss, eval_tensors, _ = create_pipeline(data_desc.eval_file, - args.max_seq_length, - args.mask_probability, - args.eval_batch_size) + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( + tokenizer, + data_file, + max_seq_length, + mask_probability, + short_seq_prob, + batch_size) + # for preprocessed data + # data_layer = nemo_nlp.BertPretrainingPreprocessedDataLayer( + # data_file, + # max_predictions_per_seq, + # batch_size, is_training) + + steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus * args.batches_per_step) + + input_data = data_layer() + + hidden_states = bert_model(input_ids=input_data.input_ids, + token_type_ids=input_data.input_type_ids, + attention_mask=input_data.input_mask) + + mlm_logits = mlm_classifier(hidden_states=hidden_states) + mlm_loss = mlm_loss_fn(logits=mlm_logits, + output_ids=input_data.output_ids, + output_mask=input_data.output_mask) + + nsp_logits = nsp_classifier(hidden_states=hidden_states) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) + + loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) + + return loss, mlm_loss, nsp_loss, steps_per_epoch + + + train_loss, _, _, steps_per_epoch = create_pipeline( + data_file=data_desc.train_file, + preprocessed_data=False, + max_seq_length=args.max_seq_length, + mask_probability=args.mask_probability, + short_seq_prob=args.short_seq_prob, + batch_size=args.batch_size, + batches_per_step=args.batches_per_step) + + # for preprocessed data + # train_loss, _, _, steps_per_epoch = create_pipeline( + # data_file=args.data_dir, + # preprocessed_data=True, + # max_predictions_per_seq=args.max_predictions_per_seq, + # training=True, + # batch_size=args.batch_size, + # batches_per_step=args.batches_per_step) + + eval_loss, eval_tensors, _ = create_pipeline(data_desc.eval_file, + args.max_seq_length, + diff --git a/docs/sources/source/nlp/asr-improvement.rst b/docs/sources/source/nlp/asr-improvement.rst index 6565b76c1f55..d8cb99de02fd 100644 --- a/docs/sources/source/nlp/asr-improvement.rst +++ b/docs/sources/source/nlp/asr-improvement.rst @@ -66,18 +66,17 @@ Then we define tokenizer to convert tokens into indices. We will use ``bert-base .. code-block:: python - tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") + tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model="bert-base-uncased") The encoder block is a neural module corresponding to BERT language model from -``nemo_nlp.huggingface`` collection: +``nemo_nlp.nm.trainables.huggingface`` collection: .. code-block:: python zeros_transform = nemo.backends.pytorch.common.ZerosLikeNM() - encoder = nemo_nlp.huggingface.BERT( - pretrained_model_name=args.pretrained_model, - local_rank=args.local_rank) + encoder = nemo_nlp.nm.trainables.huggingface.BERT( + pretrained_model_name=args.pretrained_model) .. tip:: Making embedding size (as well as all other tensor dimensions) divisible @@ -100,7 +99,7 @@ learn positional encodings ``"learn_positional_encodings": True``: .. code-block:: python - decoder = nemo_nlp.TransformerDecoderNM( + decoder = nemo_nlp.nm.trainables.TransformerDecoderNM( d_model=args.d_model, d_inner=args.d_inner, num_layers=args.num_layers, @@ -123,7 +122,7 @@ To load the pretrained parameters into decoder, we use ``restore_from`` attribut Model training -------------- -To train the model run ``asr_postprocessor.py.py`` located in ``examples/nlp`` directory. We train with novograd optimizer :cite:`asr-imps-ginsburg2019stochastic`, +To train the model run ``asr_postprocessor.py.py`` located in ``examples/nlp/asr_postprocessor`` directory. We train with novograd optimizer :cite:`asr-imps-ginsburg2019stochastic`, learning rate ``lr=0.001``, polynomial learning rate decay policy, ``1000`` warmup steps, per-gpu batch size of ``4096*8`` tokens, and ``0.25`` dropout probability. We trained on 8 GPUS. To launch the training in multi-gpu mode run the following command: diff --git a/docs/sources/source/nlp/bert_pretraining.rst b/docs/sources/source/nlp/bert_pretraining.rst index 4e409ddd2c73..2e6793874212 100644 --- a/docs/sources/source/nlp/bert_pretraining.rst +++ b/docs/sources/source/nlp/bert_pretraining.rst @@ -4,7 +4,7 @@ Pretraining BERT In this tutorial, we will build and train a masked language model, either from scratch or from a pretrained BERT model, using the BERT architecture :cite:`nlp-bert-devlin2018bert`. Make sure you have ``nemo`` and ``nemo_nlp`` installed before starting this tutorial. See the :ref:`installation` section for more details. -The code used in this tutorial can be found at ``examples/nlp/bert_pretraining.py``. +The code used in this tutorial can be found at ``examples/nlp/language_modeling/bert_pretraining.py``. Introduction ------------ @@ -77,10 +77,20 @@ To train on a Chinese dataset, you should use `NemoBertTokenizer`. # If you're using a custom vocabulary, create your tokenizer like this tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) + special_tokens = { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + # or + tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") Create the model ---------------- @@ -105,7 +115,7 @@ We also need to define the BERT model that we will be pre-training. Here, you ca .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT( + bert_model = nemo_nlp.nm.trainables.huggingface.BERT( vocab_size=args.vocab_size, num_hidden_layers=args.num_hidden_layers, hidden_size=args.hidden_size, @@ -126,22 +136,22 @@ For the full list of BERT model names, check out `nemo_nlp.huggingface.BERT.list .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT(pretrained_model_name="bert-base-cased") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name="bert-base-cased") Next, we will define our classifier and loss functions. We will demonstrate how to pre-train with both MLM (masked language model) and NSP (next sentence prediction) losses, but you may observe higher downstream accuracy by only pre-training with MLM loss. .. code-block:: python - mlm_classifier = nemo_nlp.BertTokenClassifier( + mlm_classifier = nemo_nlp.nm.trainables.BertTokenClassifier( args.hidden_size, num_classes=args.vocab_size, activation=ACT2FN[args.hidden_act], log_softmax=True) - mlm_loss_fn = nemo_nlp.MaskedLanguageModelingLossNM() + mlm_loss_fn = nemo_nlp.nm.losses.MaskedLanguageModelingLossNM() - nsp_classifier = nemo_nlp.SequenceClassifier( + nsp_classifier = nemo_nlp.nm.trainables.SequenceClassifier( args.hidden_size, num_classes=2, num_layers=2, @@ -150,7 +160,7 @@ but you may observe higher downstream accuracy by only pre-training with MLM los nsp_loss_fn = nemo.backends.pytorch.common.CrossEntropyLoss() - bert_loss = nemo_nlp.LossAggregatorNM(num_inputs=2) + bert_loss = nemo_nlp.nm.losses.LossAggregatorNM(num_inputs=2) Then, we create the pipeline from input to output that can be used for both training and evaluation: @@ -159,7 +169,7 @@ For training from raw text use nemo_nlp.BertPretrainingDataLayer, for preprocess .. code-block:: python def create_pipeline(**args): - data_layer = nemo_nlp.BertPretrainingDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( tokenizer, data_file, max_seq_length, @@ -174,20 +184,19 @@ For training from raw text use nemo_nlp.BertPretrainingDataLayer, for preprocess steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus * args.batches_per_step) - input_ids, input_type_ids, input_mask, \ - output_ids, output_mask, nsp_labels = data_layer() + input_data = data_layer() - hidden_states = bert_model(input_ids=input_ids, - token_type_ids=input_type_ids, - attention_mask=input_mask) + hidden_states = bert_model(input_ids=input_data.input_ids, + token_type_ids=input_data.input_type_ids, + attention_mask=input_data.input_mask) mlm_logits = mlm_classifier(hidden_states=hidden_states) mlm_loss = mlm_loss_fn(logits=mlm_logits, - output_ids=output_ids, - output_mask=output_mask) + output_ids=input_data.output_ids, + output_mask=input_data.output_mask) nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) diff --git a/docs/sources/source/nlp/neural_machine_translation.rst b/docs/sources/source/nlp/neural_machine_translation.rst index df5593680ba1..c62744082d66 100644 --- a/docs/sources/source/nlp/neural_machine_translation.rst +++ b/docs/sources/source/nlp/neural_machine_translation.rst @@ -3,7 +3,7 @@ Tutorial In this tutorial we are going to implement Neural Machine Translation (NMT) system based on `Transformer encoder-decoder architecture `_ :cite:`nlp-nmt-vaswani2017attention`. -All code used in this tutorial is based on ``examples/nlp/nmt_tutorial.py``. +All code used in this tutorial is based on ``examples/nlp/machine_translation/machine_translation_tutorial.py``. Preliminaries ------------- @@ -19,7 +19,7 @@ To clean the dataset we remove all sentence pairs such that: We use newstest2013 for development and newstest2014 for testing. All datasets, as well as the tokenizer model can be downloaded from `here `__. In the following steps, we assume that all data is located at ****. -**Resources.** Training script ``examples/nlp/nmt_tutorial.py`` used in this tutorial allows to train Transformer-big architecture +**Resources.** Training script ``examples/nlp/machine_translation/machine_translation_tutorial.py`` used in this tutorial allows to train Transformer-big architecture to **29.2** BLEU / **28.5** SacreBLEU on newstest2014 in approximately 15 hours on NVIDIA's DGX-1 with 16GB Volta GPUs. This setup can also be replicated with fewer resources by using more steps of gradient accumulation :cite:`nlp-nmt-ott2018scaling`. @@ -50,7 +50,7 @@ we work with 4x smaller vocabulary of 8192 BPEs. It achieves the same level of p .. code-block:: python - tokenizer = nemo_nlp.YouTokenToMeTokenizer( + tokenizer = nemo_nlp.data.YouTokenToMeTokenizer( model_path=f"{args.data_dir}/{args.src_tokenizer_model}") vocab_size = 8 * math.ceil(tokenizer.vocab_size / 8) @@ -63,9 +63,9 @@ If the source language differs from the target language a lot, then we should us .. code-block:: python - src_tokenizer = nemo_nlp.YouTokenToMeTokenizer( + src_tokenizer = nemo_nlp.data.YouTokenToMeTokenizer( model_path=f"{args.data_dir}/{args.src_tokenizer_model}") - tgt_tokenizer = nemo_nlp.CharTokenizer( + tgt_tokenizer = nemo_nlp.data.CharTokenizer( vocab_path=f"{args.data_dir}/{args.tgt_tokenizer_model}") .. tip:: @@ -80,11 +80,11 @@ Next, we define all Neural Modules necessary for our model: .. code-block:: python - encoder = nemo_nlp.TransformerEncoderNM(**encoder_params) - decoder = nemo_nlp.TransformerDecoderNM(**decoder_params) - log_softmax = nemo_nlp.TokenClassifier(**token_classifier_params) - beam_search = nemo_nlp.BeamSearchTranslatorNM(**beam_search_params) - loss = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(**loss_params) + encoder = nemo_nlp.nm.trainables.TransformerEncoderNM(**encoder_params) + decoder = nemo_nlp.nm.trainables.TransformerDecoderNM(**decoder_params) + log_softmax = nemo_nlp.nm.trainables.TokenClassifier(**token_classifier_params) + beam_search = nemo_nlp.nm.trainables.BeamSearchTranslatorNM(**beam_search_params) + loss = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM(**loss_params) Following `Press and Wolf, 2016 `_ :cite:`nlp-nmt-press2016using`, we also tie the parameters of embedding and softmax layers: @@ -103,8 +103,8 @@ in **source and target** tokens. .. code-block:: python def create_pipeline(**args): - dataset = nemo_nlp.TranslationDataset(**translation_dataset_params) - data_layer = nemo_nlp.TranslationDataLayer(dataset) + dataset = nemo_nlp.data.TranslationDataset(**translation_dataset_params) + data_layer = nemo_nlp.nm.data_layers.TranslationDataLayer(dataset) src, src_mask, tgt, tgt_mask, labels, sent_ids = data_layer() src_hiddens = encoder(input_ids=src, input_mask_src=src_mask) tgt_hiddens = decoder(input_ids_tgt=tgt, @@ -141,7 +141,7 @@ Next, we define necessary callbacks: .. code-block:: python - from nemo.collections.nlp.callbacks.translation import eval_iter_callback, eval_epochs_done_callback + from nemo.collections.nlp.callbacks.machine_translation_callbacks import eval_iter_callback, eval_epochs_done_callback train_callback = nemo.core.SimpleLossLoggerCallback(...) eval_callback = nemo.core.EvaluatorCallback(...) @@ -175,11 +175,11 @@ Finally, we define the optimization parameters and run the whole pipeline. Model training -------------- -To train the Transformer-big model, run ``nmt_tutorial.py`` located at ``nemo/examples/nlp``: +To train the Transformer-big model, run ``machine_translation_tutorial.py`` located at ``examples/nlp/machine_translation``: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node= nmt_tutorial.py \ + python -m torch.distributed.launch --nproc_per_node= machine_translation_tutorial.py \ --data_dir --src_tokenizer_model bpe8k_yttm.model \ --eval_datasets valid/newstest2013 --optimizer novograd --lr 0.04 \ --weight_decay 0.0001 --max_steps 40000 --warmup_steps 4000 \ @@ -197,9 +197,9 @@ Translation with pretrained model 1. Put your saved checkpoint (or download good checkpoint which obtains 28.5 SacreBLEU on newstest2014 from `here `__) into ****. -2. Run ``nmt_tutorial.py`` in an interactive mode:: +2. Run ``machine_translation_tutorial.py`` in an interactive mode:: - python nmt_tutorial.py --src_tokenizer_model bpe8k_yttm.model \ + python machine_translation_tutorial.py --src_tokenizer_model bpe8k_yttm.model \ --eval_datasets test --optimizer novograd --d_model 1024 \ --d_inner 4096 --num_layers 6 --num_attn_heads 16 \ --restore_checkpoint_from --interactive diff --git a/docs/sources/source/nlp/question_answering.rst b/docs/sources/source/nlp/question_answering.rst index 266e74799a7a..56c91645ed2a 100644 --- a/docs/sources/source/nlp/question_answering.rst +++ b/docs/sources/source/nlp/question_answering.rst @@ -1,14 +1,17 @@ Tutorial ======== -In this tutorial, we are going to implement a Question Answering system using the SQuAD dataset with pretrained BERT model based on +In this tutorial, we are going to implement a Question Answering system using the SQuAD dataset with pretrained BERT-like models based on `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `_ :cite:`nlp-qa-devlin2018bert`. -All code used in this tutorial is based on ``examples/nlp/squad.py``. +All code used in this tutorial is based on ``examples/nlp/question_answering/question_answering.py``. -There are four pre-trained BERT models that we can select from using the argument `--pretrained_bert_model`. We're currently -using the script for loading pre-trained models from `transformers`. See the list of available pre-trained models -`here `__. +Currently, there are 3 pretrained back-bone models supported, on which the question answering task SQuAD can be fine-tuned: +BERT, ALBERT and RoBERTa. These are pretrained model checkpoints from `transformers `__ . Apart from these, the user can also do fine-tuning +on a custom BERT checkpoint, specified by the `--bert_checkpoint` argument. +The pretrained back-bone models can be specified by `--model_type` and the specific model by `--pretrained_model_name`. +See the list of available pre-trained models +`here `__. Preliminaries ------------- @@ -36,7 +39,7 @@ This model can work with any dataset that follows the format: Currently, the datasets that we provide pre-processing script for is SQuAD v1.1 and v2.0 which can be downloaded from `https://rajpurkar.github.io/SQuAD-explorer/ `_. -You can find the pre-processing script in ``examples/nemo_nlp/scripts/download_squad.py``. +You can find the pre-processing script in ``examples/nlp/scripts/get_squad.py``. Code structure @@ -46,7 +49,9 @@ First, we instantiate Neural Module Factory which defines 1) backend (PyTorch), 3) local rank of the GPU, and 4) an experiment manager that creates a timestamped folder to store checkpoints, relevant outputs, log files, and TensorBoard graphs. .. code-block:: python - + + import nemo + import nemo.collections.nlp as nemo_nlp nf = nemo.core.NeuralModuleFactory(backend=nemo.core.Backend.PyTorch, local_rank=args.local_rank, optimization_level=args.amp_opt_level, @@ -60,36 +65,58 @@ This will tokenize text following the mapping of the original BERT model. .. code-block:: python - from nemo.collections.nlp import NemoBertTokenizer - hidden_size = pretrained_bert_model.hidden_size - tokenizer = NemoBertTokenizer(args.pretrained_bert_model) + hidden_size = model.hidden_size + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='bert', pretrained_model="bert-base-uncased") + # to use RoBERTa tokenizer, run e.g. + special_tokens_roberta = { + "unk_token": "", + "sep_token": "", + "pad_token": "", + "bos_token": "", + "mask_token": "", + "eos_token": "", + "cls_token": "", + } + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='roberta', pretrained_model="roberta-base", special_tokens=special_tokens_roberta) + # to use Albert tokenizer, run e.g. + special_tokens_albert = { + "unk_token": "", + "sep_token": "[SEP]", + "eos_token": "[SEP]", + "pad_token": "", + "cls_token": "[CLS]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + } + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='albert', pretrained_model="albert-base-v1", special_tokens=special_tokens_albert) + Next, we define all Neural Modules participating in our question answering classification pipeline. - * Process data: the `BertQuestionAnsweringDataLayer` class in ``nemo_nlp/nemo_nlp/data/data_layers.py`` is supposed to do the preprocessing of raw data into the format data supported by `SquadDataset`. + * Process data: the `BertQuestionAnsweringDataLayer` is supposed to do the preprocessing of raw data into the format data supported by `SquadDataset`. Training and evaluation each require their own `BertQuestionAnsweringDataLayer`. DataLayer is an extra layer to do the semantic checking for your dataset and convert it into DataLayerNM. .. code-block:: python - data_layer = BertQuestionAnsweringDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( + data_file=args.train_file, + tokenizer=tokenizer, + batch_size=args.batch_size, mode='train', version_2_with_negative=args.version_2_with_negative, - batch_size=args.batch_size, - tokenizer=tokenizer, - data_dir=args.data_dir, max_query_length=args.max_query_length, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride) - data_layer_eval = nemo_nlp.BertQuestionAnsweringDataLayer( + data_layer_eval = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( + data_file=args.dev_file, + tokenizer=tokenizer, + batch_size=args.batch_size, mode='dev', version_2_with_negative=args.version_2_with_negative, - batch_size=args.batch_size, - tokenizer=tokenizer, - data_dir=args.data_dir, max_query_length=args.max_query_length, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride) @@ -97,14 +124,21 @@ Next, we define all Neural Modules participating in our question answering class * Load the pretrained model and get the hidden states for the corresponding inputs. .. code-block:: python - - model = nemo_nlp.huggingface.BERT(args.pretrained_bert_model) + + args.pretrained_model_name = "bert-base-uncased" + model = nemo_nlp.nm.trainables.huggingface.BERT(args.pretrained_model_name) + # or for RoBERTa + args.pretrained_model_name = "roberta-base" + model = nemo_nlp.nm.trainables.huggingface.Roberta(args.pretrained_model_name) + # or for Albert + args.pretrained_model_name = "albert-base-v1" + model = nemo_nlp.nm.trainables.huggingface.Albert(args.pretrained_model_name) * Create the classifier head for our task. .. code-block:: python - qa_head = nemo_nlp.TokenClassifier( + qa_head = nemo_nlp.nm.trainables.TokenClassifier( hidden_size=hidden_size, num_classes=2, num_layers=1, @@ -114,7 +148,7 @@ Next, we define all Neural Modules participating in our question answering class .. code-block:: python - loss_fn = nemo_nlp.QuestionAnsweringLoss() + loss_fn = nemo_nlp.nm.losses.QuestionAnsweringLoss() * Create the pipelines for the train and evaluation processes. @@ -201,24 +235,25 @@ Next, we define all Neural Modules participating in our question answering class Model training -------------- -To train a question answering model on SQuAD using multi-gpu, run ``squad.py`` located at ``nemo/examples/nlp``: +To train a question answering model on SQuAD using multi-gpu, run ``question_answering_squad.py`` located at ``examples/nlp/question_answering``: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node=8 squad.py - --data_dir + python -m torch.distributed.launch --nproc_per_node=8 question_answering_squad.py + --train_file + --dev_file --num_gpus 8 --work_dir --amp_opt_level - --pretrained_bert_model + --pretrained_model_name ... To do inference, run: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node=8 squad.py - --data_dir + python -m torch.distributed.launch --nproc_per_node=8 question_answering_squad.py + --dev_file --num_gpus 8 --checkpoint_dir --evaluation_only @@ -230,7 +265,7 @@ To run on a single GPU, run: .. code-block:: python - python squad.py \ + python question_answering_squad.py \ ... diff --git a/examples/nlp/asr_postprocessor/asr_postprocessor.py b/examples/nlp/asr_postprocessor/asr_postprocessor.py index c9cfadd1ef73..204e9db5664f 100644 --- a/examples/nlp/asr_postprocessor/asr_postprocessor.py +++ b/examples/nlp/asr_postprocessor/asr_postprocessor.py @@ -19,14 +19,13 @@ import torch +import nemo import nemo.collections.nlp as nemo_nlp -import nemo.collections.nlp.nm.data_layers.machine_translation_datalayer from nemo import logging from nemo.collections.nlp.callbacks.machine_translation_callback import ( eval_epochs_done_callback_wer, eval_iter_callback, ) -from nemo.collections.nlp.data.tokenizers.bert_tokenizer import NemoBertTokenizer from nemo.core.callbacks import CheckpointCallback from nemo.utils.lr_policies import SquareAnnealing @@ -80,7 +79,7 @@ add_time_to_log_dir=False, ) -tokenizer = NemoBertTokenizer(pretrained_model=args.pretrained_model) +tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model=args.pretrained_model) vocab_size = 8 * math.ceil(tokenizer.vocab_size / 8) tokens_to_add = vocab_size - tokenizer.vocab_size @@ -135,7 +134,7 @@ def create_pipeline(dataset, tokens_in_batch, clean=False, training=True): dataset_src = os.path.join(args.data_dir, dataset + "." + args.src_lang) dataset_tgt = os.path.join(args.data_dir, dataset + "." + args.tgt_lang) - data_layer = nemo_nlp.nm.data_layers.machine_translation_datalayer.TranslationDataLayer( + data_layer = nemo_nlp.nm.data_layers.TranslationDataLayer( tokenizer_src=tokenizer, tokenizer_tgt=tokenizer, dataset_src=dataset_src, diff --git a/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb b/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb index f33887452dbc..b46a87a77079 100644 --- a/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb +++ b/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb @@ -15,7 +15,7 @@ "source": [ "# This example is for demonstration purposes\n", "# Please refer to the corresponding NLP tutorial on NeMo documentation\n", - "! scripts/get_wkt2.sh" + "! ../scripts/get_wkt2.sh" ] }, { @@ -35,7 +35,7 @@ "outputs": [], "source": [ "# Prepare tokenization model\n", - "! python scripts/create_vocab.py --train_path=data/lm/wikitext-2/train.txt" + "! python ../scripts/create_vocab.py --train_path=data/lm/wikitext-2/train.txt" ] }, { @@ -55,12 +55,11 @@ "import torch\n", "import nemo\n", "\n", - "from nemo.utils.lr_policies import CosineAnnealing\n", "\n", "import nemo.collections.nlp as nemo_nlp\n", - "from nemo.collections.nlp.data import NemoBertTokenizer, SentencePieceTokenizer\n", "from nemo.collections.nlp.callbacks.lm_bert_callback import eval_iter_callback, \\\n", " eval_epochs_done_callback\n", + "from nemo.utils.lr_policies import CosineAnnealing\n", "\n", "BATCHES_PER_STEP = 1\n", "BATCH_SIZE = 64\n", @@ -109,8 +108,16 @@ "outputs": [], "source": [ "# tokenizer.model file was created during Step 1\n", - "tokenizer = SentencePieceTokenizer(model_path=\"tokenizer.model\")\n", - "tokenizer.add_special_tokens([\"[MASK]\", \"[CLS]\", \"[SEP]\"])" + "tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=\"tokenizer.model\")\n", + "special_tokens = {\n", + " \"sep_token\": \"[SEP]\",\n", + " \"pad_token\": \"[PAD]\",\n", + " \"bos_token\": \"[CLS]\",\n", + " \"mask_token\": \"[MASK]\",\n", + " \"eos_token\": \"[SEP]\",\n", + " \"cls_token\": \"[CLS]\",\n", + " }\n", + "tokenizer.add_special_tokens(special_tokens)" ] }, { @@ -199,18 +206,17 @@ "outputs": [], "source": [ "# Training DAG\n", - "input_ids, input_type_ids, input_mask, \\\n", - " output_ids, output_mask, nsp_labels = train_data_layer()\n", + "input_data = train_data_layer()\n", "\n", - "hidden_states = bert_model(input_ids=input_ids,\n", - " token_type_ids=input_type_ids,\n", - " attention_mask=input_mask)\n", + "hidden_states = bert_model(input_ids=input_data.input_ids,\n", + " token_type_ids=input_data.input_type_ids,\n", + " attention_mask=input_data.input_mask)\n", "\n", "mlm_logits = mlm_classifier(hidden_states=hidden_states)\n", - "t_mlm_loss = mlm_loss(logits=mlm_logits, output_ids=output_ids, output_mask=output_mask)\n", + "t_mlm_loss = mlm_loss(logits=mlm_logits, output_ids=input_data.output_ids, output_mask=input_data.output_mask)\n", "\n", "nsp_logits = nsp_classifier(hidden_states=hidden_states)\n", - "t_nsp_loss = nsp_loss(logits=nsp_logits, labels=nsp_labels)\n", + "t_nsp_loss = nsp_loss(logits=nsp_logits, labels=input_data.labels)\n", "\n", "loss = bert_loss(loss_1=t_mlm_loss, loss_2=t_nsp_loss)" ] @@ -222,18 +228,17 @@ "outputs": [], "source": [ "# Evaluation DAG\n", - "e_input_ids, e_input_type_ids, e_input_mask, \\\n", - " e_output_ids, e_output_mask, e_nsp_labels = eval_data_layer()\n", + "input_data_eval = eval_data_layer()\n", "\n", - "e_hidden_states = bert_model(input_ids=e_input_ids,\n", - " token_type_ids=e_input_type_ids,\n", - " attention_mask=e_input_mask)\n", + "e_hidden_states = bert_model(input_ids=input_data_eval.input_ids,\n", + " token_type_ids=input_data_eval.input_type_ids,\n", + " attention_mask=input_data_eval.input_mask)\n", "\n", "e_mlm_logits = mlm_classifier(hidden_states=e_hidden_states)\n", - "e_mlm_loss = mlm_loss(logits=e_mlm_logits, output_ids=e_output_ids, output_mask=e_output_mask)\n", + "e_mlm_loss = mlm_loss(logits=e_mlm_logits, output_ids=input_data_eval.output_ids, output_mask=input_data_eval.output_mask)\n", "\n", "e_nsp_logits = nsp_classifier(hidden_states=e_hidden_states)\n", - "e_nsp_loss = nsp_loss(logits=e_nsp_logits, labels=e_nsp_labels)\n", + "e_nsp_loss = nsp_loss(logits=e_nsp_logits, labels=input_data_eval.labels)\n", "\n", "e_loss = bert_loss(loss_1=e_mlm_loss, loss_2=e_nsp_loss)" ] @@ -265,7 +270,17 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-02-12 12:08:08 callbacks:196] Step: 300\n", + "Loss: 6.991\n", + "[NeMo I 2020-02-12 12:08:08 callbacks:211] Step time: 0.13242316246032715 seconds\n" + ] + } + ], "source": [ "lr_policy = CosineAnnealing(NUM_EPOCHS * steps_per_epoch,\n", " warmup_ratio=LR_WARMUP_PROPORTION)\n", @@ -308,7 +323,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.6" } }, "nbformat": 4, diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 24b94f1fb380..4b9c201f2ddb 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -232,24 +232,26 @@ def create_pipeline(data_file, batch_size, preprocessed_data=False, batches_per_ kwargs['mask_probability'], kwargs['short_seq_prob'], ) - data_layer = nemo_nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( tokenizer, data_file, max_seq_length, mask_probability, short_seq_prob, batch_size=batch_size ) else: training, max_predictions_per_seq = (kwargs['training'], kwargs['max_predictions_per_seq']) - data_layer = nemo_nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingPreprocessedDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingPreprocessedDataLayer( data_file, max_predictions_per_seq, batch_size=batch_size, training=training ) steps_per_epoch = math.ceil(len(data_layer) / (batch_size * args.num_gpus * batches_per_step)) - (input_ids, input_type_ids, input_mask, output_ids, output_mask, nsp_labels) = data_layer() - hidden_states = bert_model(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + input_data = data_layer() + hidden_states = bert_model( + input_ids=input_data.input_ids, token_type_ids=input_data.input_type_ids, attention_mask=input_data.input_mask + ) mlm_logits = mlm_classifier(hidden_states=hidden_states) - mlm_loss = mlm_loss_fn(logits=mlm_logits, output_ids=output_ids, output_mask=output_mask) + mlm_loss = mlm_loss_fn(logits=mlm_logits, output_ids=input_data.output_ids, output_mask=input_data.output_mask) if not args.only_mlm_loss: nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) else: loss = mlm_loss diff --git a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py index f8467eff6d51..8cda90810521 100644 --- a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py +++ b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py @@ -146,7 +146,7 @@ max_seq_length=args.max_seq_length, ) -log_softmax = nemo_nlp.nm.trainables.token_classification_nm.TokenClassifier( +log_softmax = nemo_nlp.nm.trainables.TokenClassifier( args.d_model, num_classes=tgt_tokenizer.vocab_size, num_layers=1, log_softmax=True ) diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 17bccbf5cac6..630bd46939f7 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -216,7 +216,7 @@ def create_pipeline( batches_per_step=1, mode="train", ): - data_layer = nemo_nlp.nm.data_layers.qa_squad_datalayer.BertQuestionAnsweringDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( mode=mode, version_2_with_negative=version_2_with_negative, batch_size=batch_size, @@ -316,7 +316,7 @@ def create_pipeline( if args.tokenizer == "sentencepiece": try: - tokenizer = nemo_nlp.SentencePieceTokenizer(model_path=args.tokenizer_model) + tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=args.tokenizer_model) except Exception: raise ValueError( "Using --tokenizer=sentencepiece \ @@ -354,13 +354,13 @@ def create_pipeline( else: """ Use this if you're using a standard BERT model. To see the list of pretrained models, call: - nemo_nlp.huggingface.BERT.list_pretrained_models() + nemo_nlp.nm.trainables.huggingface.BERT.list_pretrained_models() """ model = model_cls(pretrained_model_name=args.pretrained_model_name) hidden_size = model.hidden_size - qa_head = nemo_nlp.nm.trainables.token_classification_nm.TokenClassifier( + qa_head = nemo_nlp.nm.trainables.TokenClassifier( hidden_size=hidden_size, num_classes=2, num_layers=1, log_softmax=False ) squad_loss = nemo_nlp.nm.losses.QuestionAnsweringLoss() From 077f98947deeb65ab6f0e5142b5fc774a17603e5 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 12:14:39 -0800 Subject: [PATCH 50/70] updated nlp documentation for bert, squad, asr_post_processing and prettified module usage Signed-off-by: Yang Zhang --- .../sources/source/nlp/bert_pretraining.rst | 127 +++++++++++------- docs/sources/source/nlp/asr-improvement.rst | 13 +- docs/sources/source/nlp/bert_pretraining.rst | 43 +++--- .../source/nlp/neural_machine_translation.rst | 34 ++--- .../sources/source/nlp/question_answering.rst | 95 ++++++++----- .../asr_postprocessor/asr_postprocessor.py | 7 +- .../BERTPretrainingTutorial.ipynb | 59 +++++--- .../nlp/language_modeling/bert_pretraining.py | 16 ++- .../machine_translation_tutorial.py | 2 +- .../question_answering_squad.py | 8 +- 10 files changed, 248 insertions(+), 156 deletions(-) diff --git a/docs/docs_zh/sources/source/nlp/bert_pretraining.rst b/docs/docs_zh/sources/source/nlp/bert_pretraining.rst index e3877298f027..24d3a556020b 100644 --- a/docs/docs_zh/sources/source/nlp/bert_pretraining.rst +++ b/docs/docs_zh/sources/source/nlp/bert_pretraining.rst @@ -5,7 +5,7 @@ BERT预训练 创建一个专门领域的BERT模型对于某些应用是更有优势的。比如一个专门针对生物医学领域的专业BERT，类似于BioBERT :cite:`nlp-bert-lee2019biobert` 和SciBERT :cite:`nlp-bert-beltagy2019scibert` 。 -本教程中所使用的代码来自于 ``examples/nlp/bert_pretraining.py``. +本教程中所使用的代码来自于 ``examples/nlp/language_modeling/bert_pretraining.py``. 语料下载 -------- @@ -51,10 +51,20 @@ BERT预训练 # If you're using a custom vocabulary, create your tokenizer like this tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) + special_tokens = { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + # or + tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") 创建模型 -------- @@ -78,76 +88,99 @@ BERT预训练 .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT( - vocab_size=tokenizer.vocab_size, - num_layers=args.num_layers, - d_model=args.d_model, - num_heads=args.num_heads, - d_inner=args.d_inner, - max_seq_length=args.max_seq_length, - hidden_act="gelu") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT( + vocab_size=args.vocab_size, + num_hidden_layers=args.num_hidden_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + intermediate_size=args.intermediate_size, + max_position_embeddings=args.max_seq_length, + hidden_act=args.hidden_act) 如果你想从一个已有的BERT模型文件继续训练，那设置一个模型的名字即可。如果想查看完整的预训练好的BERT模型列表，可以使用 `nemo_nlp.huggingface.BERT.list_pretrained_models()` 。 .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT(pretrained_model_name="bert-base-cased") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name="bert-base-cased") 接下来，我们需要定义分类器和损失函数。在本教程中，我们会同时使用掩码语言模型和预测下一句模型这两个模型的损失函数，如果你只用掩饰语言模型作为损失的话，可能会观察到更高的准确率。 .. code-block:: python - mlm_classifier = nemo_nlp.TokenClassifier(args.d_model, + mlm_classifier = nemo_nlp.nm.trainables.TokenClassifier(args.d_model, num_classes=tokenizer.vocab_size, num_layers=1, log_softmax=True) - mlm_loss_fn = nemo_nlp.MaskedLanguageModelingLossNM() + mlm_loss_fn = nemo_nlp.nm.losses.MaskedLanguageModelingLossNM() - nsp_classifier = nemo_nlp.SequenceClassifier(args.d_model, + nsp_classifier = nemo_nlp.nm.trainables.SequenceClassifier(args.d_model, num_classes=2, num_layers=2, log_softmax=True) nsp_loss_fn = nemo.backends.pytorch.common.CrossEntropyLoss() - bert_loss = nemo_nlp.LossAggregatorNM(num_inputs=2) + bert_loss = nemo_nlp.nm.losses.LossAggregatorNM(num_inputs=2) 然后，我们把从输入到输出的整个计算流程封装成一个函数。有了这个函数，我们就可以很方便的分别创建训练流和评估流： .. code-block:: python def create_pipeline(**args): - dataset = nemo_nlp.BertPretrainingDataset(**params) - data_layer = nemo_nlp.BertPretrainingDataLayer(dataset) - steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus) - - input_ids, input_type_ids, input_mask, \ - output_ids, output_mask, nsp_labels = data_layer() - - hidden_states = bert_model(input_ids=input_ids, - token_type_ids=input_type_ids, - attention_mask=input_mask) - - mlm_logits = mlm_classifier(hidden_states=hidden_states) - mlm_loss = mlm_loss_fn(logits=mlm_logits, - output_ids=output_ids, - output_mask=output_mask) - - nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) - - loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) - - return loss, [mlm_loss, nsp_loss], steps_per_epoch - - - train_loss, _, steps_per_epoch = create_pipeline(data_desc.train_file, - args.max_seq_length, - args.mask_probability, - args.batch_size) - eval_loss, eval_tensors, _ = create_pipeline(data_desc.eval_file, - args.max_seq_length, - args.mask_probability, - args.eval_batch_size) + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( + tokenizer, + data_file, + max_seq_length, + mask_probability, + short_seq_prob, + batch_size) + # for preprocessed data + # data_layer = nemo_nlp.BertPretrainingPreprocessedDataLayer( + # data_file, + # max_predictions_per_seq, + # batch_size, is_training) + + steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus * args.batches_per_step) + + input_data = data_layer() + + hidden_states = bert_model(input_ids=input_data.input_ids, + token_type_ids=input_data.input_type_ids, + attention_mask=input_data.input_mask) + + mlm_logits = mlm_classifier(hidden_states=hidden_states) + mlm_loss = mlm_loss_fn(logits=mlm_logits, + output_ids=input_data.output_ids, + output_mask=input_data.output_mask) + + nsp_logits = nsp_classifier(hidden_states=hidden_states) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) + + loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) + + return loss, mlm_loss, nsp_loss, steps_per_epoch + + + train_loss, _, _, steps_per_epoch = create_pipeline( + data_file=data_desc.train_file, + preprocessed_data=False, + max_seq_length=args.max_seq_length, + mask_probability=args.mask_probability, + short_seq_prob=args.short_seq_prob, + batch_size=args.batch_size, + batches_per_step=args.batches_per_step) + + # for preprocessed data + # train_loss, _, _, steps_per_epoch = create_pipeline( + # data_file=args.data_dir, + # preprocessed_data=True, + # max_predictions_per_seq=args.max_predictions_per_seq, + # training=True, + # batch_size=args.batch_size, + # batches_per_step=args.batches_per_step) + + eval_loss, eval_tensors, _ = create_pipeline(data_desc.eval_file, + args.max_seq_length, + diff --git a/docs/sources/source/nlp/asr-improvement.rst b/docs/sources/source/nlp/asr-improvement.rst index 6565b76c1f55..d8cb99de02fd 100644 --- a/docs/sources/source/nlp/asr-improvement.rst +++ b/docs/sources/source/nlp/asr-improvement.rst @@ -66,18 +66,17 @@ Then we define tokenizer to convert tokens into indices. We will use ``bert-base .. code-block:: python - tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") + tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model="bert-base-uncased") The encoder block is a neural module corresponding to BERT language model from -``nemo_nlp.huggingface`` collection: +``nemo_nlp.nm.trainables.huggingface`` collection: .. code-block:: python zeros_transform = nemo.backends.pytorch.common.ZerosLikeNM() - encoder = nemo_nlp.huggingface.BERT( - pretrained_model_name=args.pretrained_model, - local_rank=args.local_rank) + encoder = nemo_nlp.nm.trainables.huggingface.BERT( + pretrained_model_name=args.pretrained_model) .. tip:: Making embedding size (as well as all other tensor dimensions) divisible @@ -100,7 +99,7 @@ learn positional encodings ``"learn_positional_encodings": True``: .. code-block:: python - decoder = nemo_nlp.TransformerDecoderNM( + decoder = nemo_nlp.nm.trainables.TransformerDecoderNM( d_model=args.d_model, d_inner=args.d_inner, num_layers=args.num_layers, @@ -123,7 +122,7 @@ To load the pretrained parameters into decoder, we use ``restore_from`` attribut Model training -------------- -To train the model run ``asr_postprocessor.py.py`` located in ``examples/nlp`` directory. We train with novograd optimizer :cite:`asr-imps-ginsburg2019stochastic`, +To train the model run ``asr_postprocessor.py.py`` located in ``examples/nlp/asr_postprocessor`` directory. We train with novograd optimizer :cite:`asr-imps-ginsburg2019stochastic`, learning rate ``lr=0.001``, polynomial learning rate decay policy, ``1000`` warmup steps, per-gpu batch size of ``4096*8`` tokens, and ``0.25`` dropout probability. We trained on 8 GPUS. To launch the training in multi-gpu mode run the following command: diff --git a/docs/sources/source/nlp/bert_pretraining.rst b/docs/sources/source/nlp/bert_pretraining.rst index 4e409ddd2c73..2e6793874212 100644 --- a/docs/sources/source/nlp/bert_pretraining.rst +++ b/docs/sources/source/nlp/bert_pretraining.rst @@ -4,7 +4,7 @@ Pretraining BERT In this tutorial, we will build and train a masked language model, either from scratch or from a pretrained BERT model, using the BERT architecture :cite:`nlp-bert-devlin2018bert`. Make sure you have ``nemo`` and ``nemo_nlp`` installed before starting this tutorial. See the :ref:`installation` section for more details. -The code used in this tutorial can be found at ``examples/nlp/bert_pretraining.py``. +The code used in this tutorial can be found at ``examples/nlp/language_modeling/bert_pretraining.py``. Introduction ------------ @@ -77,10 +77,20 @@ To train on a Chinese dataset, you should use `NemoBertTokenizer`. # If you're using a custom vocabulary, create your tokenizer like this tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) + special_tokens = { + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + } + tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + # or + tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") Create the model ---------------- @@ -105,7 +115,7 @@ We also need to define the BERT model that we will be pre-training. Here, you ca .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT( + bert_model = nemo_nlp.nm.trainables.huggingface.BERT( vocab_size=args.vocab_size, num_hidden_layers=args.num_hidden_layers, hidden_size=args.hidden_size, @@ -126,22 +136,22 @@ For the full list of BERT model names, check out `nemo_nlp.huggingface.BERT.list .. code-block:: python - bert_model = nemo_nlp.huggingface.BERT(pretrained_model_name="bert-base-cased") + bert_model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name="bert-base-cased") Next, we will define our classifier and loss functions. We will demonstrate how to pre-train with both MLM (masked language model) and NSP (next sentence prediction) losses, but you may observe higher downstream accuracy by only pre-training with MLM loss. .. code-block:: python - mlm_classifier = nemo_nlp.BertTokenClassifier( + mlm_classifier = nemo_nlp.nm.trainables.BertTokenClassifier( args.hidden_size, num_classes=args.vocab_size, activation=ACT2FN[args.hidden_act], log_softmax=True) - mlm_loss_fn = nemo_nlp.MaskedLanguageModelingLossNM() + mlm_loss_fn = nemo_nlp.nm.losses.MaskedLanguageModelingLossNM() - nsp_classifier = nemo_nlp.SequenceClassifier( + nsp_classifier = nemo_nlp.nm.trainables.SequenceClassifier( args.hidden_size, num_classes=2, num_layers=2, @@ -150,7 +160,7 @@ but you may observe higher downstream accuracy by only pre-training with MLM los nsp_loss_fn = nemo.backends.pytorch.common.CrossEntropyLoss() - bert_loss = nemo_nlp.LossAggregatorNM(num_inputs=2) + bert_loss = nemo_nlp.nm.losses.LossAggregatorNM(num_inputs=2) Then, we create the pipeline from input to output that can be used for both training and evaluation: @@ -159,7 +169,7 @@ For training from raw text use nemo_nlp.BertPretrainingDataLayer, for preprocess .. code-block:: python def create_pipeline(**args): - data_layer = nemo_nlp.BertPretrainingDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( tokenizer, data_file, max_seq_length, @@ -174,20 +184,19 @@ For training from raw text use nemo_nlp.BertPretrainingDataLayer, for preprocess steps_per_epoch = len(data_layer) // (batch_size * args.num_gpus * args.batches_per_step) - input_ids, input_type_ids, input_mask, \ - output_ids, output_mask, nsp_labels = data_layer() + input_data = data_layer() - hidden_states = bert_model(input_ids=input_ids, - token_type_ids=input_type_ids, - attention_mask=input_mask) + hidden_states = bert_model(input_ids=input_data.input_ids, + token_type_ids=input_data.input_type_ids, + attention_mask=input_data.input_mask) mlm_logits = mlm_classifier(hidden_states=hidden_states) mlm_loss = mlm_loss_fn(logits=mlm_logits, - output_ids=output_ids, - output_mask=output_mask) + output_ids=input_data.output_ids, + output_mask=input_data.output_mask) nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) diff --git a/docs/sources/source/nlp/neural_machine_translation.rst b/docs/sources/source/nlp/neural_machine_translation.rst index df5593680ba1..c62744082d66 100644 --- a/docs/sources/source/nlp/neural_machine_translation.rst +++ b/docs/sources/source/nlp/neural_machine_translation.rst @@ -3,7 +3,7 @@ Tutorial In this tutorial we are going to implement Neural Machine Translation (NMT) system based on `Transformer encoder-decoder architecture `_ :cite:`nlp-nmt-vaswani2017attention`. -All code used in this tutorial is based on ``examples/nlp/nmt_tutorial.py``. +All code used in this tutorial is based on ``examples/nlp/machine_translation/machine_translation_tutorial.py``. Preliminaries ------------- @@ -19,7 +19,7 @@ To clean the dataset we remove all sentence pairs such that: We use newstest2013 for development and newstest2014 for testing. All datasets, as well as the tokenizer model can be downloaded from `here `__. In the following steps, we assume that all data is located at ****. -**Resources.** Training script ``examples/nlp/nmt_tutorial.py`` used in this tutorial allows to train Transformer-big architecture +**Resources.** Training script ``examples/nlp/machine_translation/machine_translation_tutorial.py`` used in this tutorial allows to train Transformer-big architecture to **29.2** BLEU / **28.5** SacreBLEU on newstest2014 in approximately 15 hours on NVIDIA's DGX-1 with 16GB Volta GPUs. This setup can also be replicated with fewer resources by using more steps of gradient accumulation :cite:`nlp-nmt-ott2018scaling`. @@ -50,7 +50,7 @@ we work with 4x smaller vocabulary of 8192 BPEs. It achieves the same level of p .. code-block:: python - tokenizer = nemo_nlp.YouTokenToMeTokenizer( + tokenizer = nemo_nlp.data.YouTokenToMeTokenizer( model_path=f"{args.data_dir}/{args.src_tokenizer_model}") vocab_size = 8 * math.ceil(tokenizer.vocab_size / 8) @@ -63,9 +63,9 @@ If the source language differs from the target language a lot, then we should us .. code-block:: python - src_tokenizer = nemo_nlp.YouTokenToMeTokenizer( + src_tokenizer = nemo_nlp.data.YouTokenToMeTokenizer( model_path=f"{args.data_dir}/{args.src_tokenizer_model}") - tgt_tokenizer = nemo_nlp.CharTokenizer( + tgt_tokenizer = nemo_nlp.data.CharTokenizer( vocab_path=f"{args.data_dir}/{args.tgt_tokenizer_model}") .. tip:: @@ -80,11 +80,11 @@ Next, we define all Neural Modules necessary for our model: .. code-block:: python - encoder = nemo_nlp.TransformerEncoderNM(**encoder_params) - decoder = nemo_nlp.TransformerDecoderNM(**decoder_params) - log_softmax = nemo_nlp.TokenClassifier(**token_classifier_params) - beam_search = nemo_nlp.BeamSearchTranslatorNM(**beam_search_params) - loss = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(**loss_params) + encoder = nemo_nlp.nm.trainables.TransformerEncoderNM(**encoder_params) + decoder = nemo_nlp.nm.trainables.TransformerDecoderNM(**decoder_params) + log_softmax = nemo_nlp.nm.trainables.TokenClassifier(**token_classifier_params) + beam_search = nemo_nlp.nm.trainables.BeamSearchTranslatorNM(**beam_search_params) + loss = nemo_nlp.nm.losses.PaddedSmoothedCrossEntropyLossNM(**loss_params) Following `Press and Wolf, 2016 `_ :cite:`nlp-nmt-press2016using`, we also tie the parameters of embedding and softmax layers: @@ -103,8 +103,8 @@ in **source and target** tokens. .. code-block:: python def create_pipeline(**args): - dataset = nemo_nlp.TranslationDataset(**translation_dataset_params) - data_layer = nemo_nlp.TranslationDataLayer(dataset) + dataset = nemo_nlp.data.TranslationDataset(**translation_dataset_params) + data_layer = nemo_nlp.nm.data_layers.TranslationDataLayer(dataset) src, src_mask, tgt, tgt_mask, labels, sent_ids = data_layer() src_hiddens = encoder(input_ids=src, input_mask_src=src_mask) tgt_hiddens = decoder(input_ids_tgt=tgt, @@ -141,7 +141,7 @@ Next, we define necessary callbacks: .. code-block:: python - from nemo.collections.nlp.callbacks.translation import eval_iter_callback, eval_epochs_done_callback + from nemo.collections.nlp.callbacks.machine_translation_callbacks import eval_iter_callback, eval_epochs_done_callback train_callback = nemo.core.SimpleLossLoggerCallback(...) eval_callback = nemo.core.EvaluatorCallback(...) @@ -175,11 +175,11 @@ Finally, we define the optimization parameters and run the whole pipeline. Model training -------------- -To train the Transformer-big model, run ``nmt_tutorial.py`` located at ``nemo/examples/nlp``: +To train the Transformer-big model, run ``machine_translation_tutorial.py`` located at ``examples/nlp/machine_translation``: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node= nmt_tutorial.py \ + python -m torch.distributed.launch --nproc_per_node= machine_translation_tutorial.py \ --data_dir --src_tokenizer_model bpe8k_yttm.model \ --eval_datasets valid/newstest2013 --optimizer novograd --lr 0.04 \ --weight_decay 0.0001 --max_steps 40000 --warmup_steps 4000 \ @@ -197,9 +197,9 @@ Translation with pretrained model 1. Put your saved checkpoint (or download good checkpoint which obtains 28.5 SacreBLEU on newstest2014 from `here `__) into ****. -2. Run ``nmt_tutorial.py`` in an interactive mode:: +2. Run ``machine_translation_tutorial.py`` in an interactive mode:: - python nmt_tutorial.py --src_tokenizer_model bpe8k_yttm.model \ + python machine_translation_tutorial.py --src_tokenizer_model bpe8k_yttm.model \ --eval_datasets test --optimizer novograd --d_model 1024 \ --d_inner 4096 --num_layers 6 --num_attn_heads 16 \ --restore_checkpoint_from --interactive diff --git a/docs/sources/source/nlp/question_answering.rst b/docs/sources/source/nlp/question_answering.rst index 266e74799a7a..56c91645ed2a 100644 --- a/docs/sources/source/nlp/question_answering.rst +++ b/docs/sources/source/nlp/question_answering.rst @@ -1,14 +1,17 @@ Tutorial ======== -In this tutorial, we are going to implement a Question Answering system using the SQuAD dataset with pretrained BERT model based on +In this tutorial, we are going to implement a Question Answering system using the SQuAD dataset with pretrained BERT-like models based on `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `_ :cite:`nlp-qa-devlin2018bert`. -All code used in this tutorial is based on ``examples/nlp/squad.py``. +All code used in this tutorial is based on ``examples/nlp/question_answering/question_answering.py``. -There are four pre-trained BERT models that we can select from using the argument `--pretrained_bert_model`. We're currently -using the script for loading pre-trained models from `transformers`. See the list of available pre-trained models -`here `__. +Currently, there are 3 pretrained back-bone models supported, on which the question answering task SQuAD can be fine-tuned: +BERT, ALBERT and RoBERTa. These are pretrained model checkpoints from `transformers `__ . Apart from these, the user can also do fine-tuning +on a custom BERT checkpoint, specified by the `--bert_checkpoint` argument. +The pretrained back-bone models can be specified by `--model_type` and the specific model by `--pretrained_model_name`. +See the list of available pre-trained models +`here `__. Preliminaries ------------- @@ -36,7 +39,7 @@ This model can work with any dataset that follows the format: Currently, the datasets that we provide pre-processing script for is SQuAD v1.1 and v2.0 which can be downloaded from `https://rajpurkar.github.io/SQuAD-explorer/ `_. -You can find the pre-processing script in ``examples/nemo_nlp/scripts/download_squad.py``. +You can find the pre-processing script in ``examples/nlp/scripts/get_squad.py``. Code structure @@ -46,7 +49,9 @@ First, we instantiate Neural Module Factory which defines 1) backend (PyTorch), 3) local rank of the GPU, and 4) an experiment manager that creates a timestamped folder to store checkpoints, relevant outputs, log files, and TensorBoard graphs. .. code-block:: python - + + import nemo + import nemo.collections.nlp as nemo_nlp nf = nemo.core.NeuralModuleFactory(backend=nemo.core.Backend.PyTorch, local_rank=args.local_rank, optimization_level=args.amp_opt_level, @@ -60,36 +65,58 @@ This will tokenize text following the mapping of the original BERT model. .. code-block:: python - from nemo.collections.nlp import NemoBertTokenizer - hidden_size = pretrained_bert_model.hidden_size - tokenizer = NemoBertTokenizer(args.pretrained_bert_model) + hidden_size = model.hidden_size + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='bert', pretrained_model="bert-base-uncased") + # to use RoBERTa tokenizer, run e.g. + special_tokens_roberta = { + "unk_token": "", + "sep_token": "", + "pad_token": "", + "bos_token": "", + "mask_token": "", + "eos_token": "", + "cls_token": "", + } + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='roberta', pretrained_model="roberta-base", special_tokens=special_tokens_roberta) + # to use Albert tokenizer, run e.g. + special_tokens_albert = { + "unk_token": "", + "sep_token": "[SEP]", + "eos_token": "[SEP]", + "pad_token": "", + "cls_token": "[CLS]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + } + tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='albert', pretrained_model="albert-base-v1", special_tokens=special_tokens_albert) + Next, we define all Neural Modules participating in our question answering classification pipeline. - * Process data: the `BertQuestionAnsweringDataLayer` class in ``nemo_nlp/nemo_nlp/data/data_layers.py`` is supposed to do the preprocessing of raw data into the format data supported by `SquadDataset`. + * Process data: the `BertQuestionAnsweringDataLayer` is supposed to do the preprocessing of raw data into the format data supported by `SquadDataset`. Training and evaluation each require their own `BertQuestionAnsweringDataLayer`. DataLayer is an extra layer to do the semantic checking for your dataset and convert it into DataLayerNM. .. code-block:: python - data_layer = BertQuestionAnsweringDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( + data_file=args.train_file, + tokenizer=tokenizer, + batch_size=args.batch_size, mode='train', version_2_with_negative=args.version_2_with_negative, - batch_size=args.batch_size, - tokenizer=tokenizer, - data_dir=args.data_dir, max_query_length=args.max_query_length, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride) - data_layer_eval = nemo_nlp.BertQuestionAnsweringDataLayer( + data_layer_eval = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( + data_file=args.dev_file, + tokenizer=tokenizer, + batch_size=args.batch_size, mode='dev', version_2_with_negative=args.version_2_with_negative, - batch_size=args.batch_size, - tokenizer=tokenizer, - data_dir=args.data_dir, max_query_length=args.max_query_length, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride) @@ -97,14 +124,21 @@ Next, we define all Neural Modules participating in our question answering class * Load the pretrained model and get the hidden states for the corresponding inputs. .. code-block:: python - - model = nemo_nlp.huggingface.BERT(args.pretrained_bert_model) + + args.pretrained_model_name = "bert-base-uncased" + model = nemo_nlp.nm.trainables.huggingface.BERT(args.pretrained_model_name) + # or for RoBERTa + args.pretrained_model_name = "roberta-base" + model = nemo_nlp.nm.trainables.huggingface.Roberta(args.pretrained_model_name) + # or for Albert + args.pretrained_model_name = "albert-base-v1" + model = nemo_nlp.nm.trainables.huggingface.Albert(args.pretrained_model_name) * Create the classifier head for our task. .. code-block:: python - qa_head = nemo_nlp.TokenClassifier( + qa_head = nemo_nlp.nm.trainables.TokenClassifier( hidden_size=hidden_size, num_classes=2, num_layers=1, @@ -114,7 +148,7 @@ Next, we define all Neural Modules participating in our question answering class .. code-block:: python - loss_fn = nemo_nlp.QuestionAnsweringLoss() + loss_fn = nemo_nlp.nm.losses.QuestionAnsweringLoss() * Create the pipelines for the train and evaluation processes. @@ -201,24 +235,25 @@ Next, we define all Neural Modules participating in our question answering class Model training -------------- -To train a question answering model on SQuAD using multi-gpu, run ``squad.py`` located at ``nemo/examples/nlp``: +To train a question answering model on SQuAD using multi-gpu, run ``question_answering_squad.py`` located at ``examples/nlp/question_answering``: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node=8 squad.py - --data_dir + python -m torch.distributed.launch --nproc_per_node=8 question_answering_squad.py + --train_file + --dev_file --num_gpus 8 --work_dir --amp_opt_level - --pretrained_bert_model + --pretrained_model_name ... To do inference, run: .. code-block:: python - python -m torch.distributed.launch --nproc_per_node=8 squad.py - --data_dir + python -m torch.distributed.launch --nproc_per_node=8 question_answering_squad.py + --dev_file --num_gpus 8 --checkpoint_dir --evaluation_only @@ -230,7 +265,7 @@ To run on a single GPU, run: .. code-block:: python - python squad.py \ + python question_answering_squad.py \ ... diff --git a/examples/nlp/asr_postprocessor/asr_postprocessor.py b/examples/nlp/asr_postprocessor/asr_postprocessor.py index c9cfadd1ef73..204e9db5664f 100644 --- a/examples/nlp/asr_postprocessor/asr_postprocessor.py +++ b/examples/nlp/asr_postprocessor/asr_postprocessor.py @@ -19,14 +19,13 @@ import torch +import nemo import nemo.collections.nlp as nemo_nlp -import nemo.collections.nlp.nm.data_layers.machine_translation_datalayer from nemo import logging from nemo.collections.nlp.callbacks.machine_translation_callback import ( eval_epochs_done_callback_wer, eval_iter_callback, ) -from nemo.collections.nlp.data.tokenizers.bert_tokenizer import NemoBertTokenizer from nemo.core.callbacks import CheckpointCallback from nemo.utils.lr_policies import SquareAnnealing @@ -80,7 +79,7 @@ add_time_to_log_dir=False, ) -tokenizer = NemoBertTokenizer(pretrained_model=args.pretrained_model) +tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model=args.pretrained_model) vocab_size = 8 * math.ceil(tokenizer.vocab_size / 8) tokens_to_add = vocab_size - tokenizer.vocab_size @@ -135,7 +134,7 @@ def create_pipeline(dataset, tokens_in_batch, clean=False, training=True): dataset_src = os.path.join(args.data_dir, dataset + "." + args.src_lang) dataset_tgt = os.path.join(args.data_dir, dataset + "." + args.tgt_lang) - data_layer = nemo_nlp.nm.data_layers.machine_translation_datalayer.TranslationDataLayer( + data_layer = nemo_nlp.nm.data_layers.TranslationDataLayer( tokenizer_src=tokenizer, tokenizer_tgt=tokenizer, dataset_src=dataset_src, diff --git a/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb b/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb index f33887452dbc..b46a87a77079 100644 --- a/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb +++ b/examples/nlp/language_modeling/BERTPretrainingTutorial.ipynb @@ -15,7 +15,7 @@ "source": [ "# This example is for demonstration purposes\n", "# Please refer to the corresponding NLP tutorial on NeMo documentation\n", - "! scripts/get_wkt2.sh" + "! ../scripts/get_wkt2.sh" ] }, { @@ -35,7 +35,7 @@ "outputs": [], "source": [ "# Prepare tokenization model\n", - "! python scripts/create_vocab.py --train_path=data/lm/wikitext-2/train.txt" + "! python ../scripts/create_vocab.py --train_path=data/lm/wikitext-2/train.txt" ] }, { @@ -55,12 +55,11 @@ "import torch\n", "import nemo\n", "\n", - "from nemo.utils.lr_policies import CosineAnnealing\n", "\n", "import nemo.collections.nlp as nemo_nlp\n", - "from nemo.collections.nlp.data import NemoBertTokenizer, SentencePieceTokenizer\n", "from nemo.collections.nlp.callbacks.lm_bert_callback import eval_iter_callback, \\\n", " eval_epochs_done_callback\n", + "from nemo.utils.lr_policies import CosineAnnealing\n", "\n", "BATCHES_PER_STEP = 1\n", "BATCH_SIZE = 64\n", @@ -109,8 +108,16 @@ "outputs": [], "source": [ "# tokenizer.model file was created during Step 1\n", - "tokenizer = SentencePieceTokenizer(model_path=\"tokenizer.model\")\n", - "tokenizer.add_special_tokens([\"[MASK]\", \"[CLS]\", \"[SEP]\"])" + "tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=\"tokenizer.model\")\n", + "special_tokens = {\n", + " \"sep_token\": \"[SEP]\",\n", + " \"pad_token\": \"[PAD]\",\n", + " \"bos_token\": \"[CLS]\",\n", + " \"mask_token\": \"[MASK]\",\n", + " \"eos_token\": \"[SEP]\",\n", + " \"cls_token\": \"[CLS]\",\n", + " }\n", + "tokenizer.add_special_tokens(special_tokens)" ] }, { @@ -199,18 +206,17 @@ "outputs": [], "source": [ "# Training DAG\n", - "input_ids, input_type_ids, input_mask, \\\n", - " output_ids, output_mask, nsp_labels = train_data_layer()\n", + "input_data = train_data_layer()\n", "\n", - "hidden_states = bert_model(input_ids=input_ids,\n", - " token_type_ids=input_type_ids,\n", - " attention_mask=input_mask)\n", + "hidden_states = bert_model(input_ids=input_data.input_ids,\n", + " token_type_ids=input_data.input_type_ids,\n", + " attention_mask=input_data.input_mask)\n", "\n", "mlm_logits = mlm_classifier(hidden_states=hidden_states)\n", - "t_mlm_loss = mlm_loss(logits=mlm_logits, output_ids=output_ids, output_mask=output_mask)\n", + "t_mlm_loss = mlm_loss(logits=mlm_logits, output_ids=input_data.output_ids, output_mask=input_data.output_mask)\n", "\n", "nsp_logits = nsp_classifier(hidden_states=hidden_states)\n", - "t_nsp_loss = nsp_loss(logits=nsp_logits, labels=nsp_labels)\n", + "t_nsp_loss = nsp_loss(logits=nsp_logits, labels=input_data.labels)\n", "\n", "loss = bert_loss(loss_1=t_mlm_loss, loss_2=t_nsp_loss)" ] @@ -222,18 +228,17 @@ "outputs": [], "source": [ "# Evaluation DAG\n", - "e_input_ids, e_input_type_ids, e_input_mask, \\\n", - " e_output_ids, e_output_mask, e_nsp_labels = eval_data_layer()\n", + "input_data_eval = eval_data_layer()\n", "\n", - "e_hidden_states = bert_model(input_ids=e_input_ids,\n", - " token_type_ids=e_input_type_ids,\n", - " attention_mask=e_input_mask)\n", + "e_hidden_states = bert_model(input_ids=input_data_eval.input_ids,\n", + " token_type_ids=input_data_eval.input_type_ids,\n", + " attention_mask=input_data_eval.input_mask)\n", "\n", "e_mlm_logits = mlm_classifier(hidden_states=e_hidden_states)\n", - "e_mlm_loss = mlm_loss(logits=e_mlm_logits, output_ids=e_output_ids, output_mask=e_output_mask)\n", + "e_mlm_loss = mlm_loss(logits=e_mlm_logits, output_ids=input_data_eval.output_ids, output_mask=input_data_eval.output_mask)\n", "\n", "e_nsp_logits = nsp_classifier(hidden_states=e_hidden_states)\n", - "e_nsp_loss = nsp_loss(logits=e_nsp_logits, labels=e_nsp_labels)\n", + "e_nsp_loss = nsp_loss(logits=e_nsp_logits, labels=input_data_eval.labels)\n", "\n", "e_loss = bert_loss(loss_1=e_mlm_loss, loss_2=e_nsp_loss)" ] @@ -265,7 +270,17 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-02-12 12:08:08 callbacks:196] Step: 300\n", + "Loss: 6.991\n", + "[NeMo I 2020-02-12 12:08:08 callbacks:211] Step time: 0.13242316246032715 seconds\n" + ] + } + ], "source": [ "lr_policy = CosineAnnealing(NUM_EPOCHS * steps_per_epoch,\n", " warmup_ratio=LR_WARMUP_PROPORTION)\n", @@ -308,7 +323,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.6" } }, "nbformat": 4, diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 24b94f1fb380..0273615d2b38 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -189,7 +189,7 @@ raise ValueError("Please add your tokenizer " "or use sentence-piece or nemo-bert.") args.vocab_size = tokenizer.vocab_size -(vars(args)) + bert_model = nemo_nlp.nm.trainables.huggingface.BERT( vocab_size=args.vocab_size, num_hidden_layers=args.num_hidden_layers, @@ -232,24 +232,26 @@ def create_pipeline(data_file, batch_size, preprocessed_data=False, batches_per_ kwargs['mask_probability'], kwargs['short_seq_prob'], ) - data_layer = nemo_nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingDataLayer( tokenizer, data_file, max_seq_length, mask_probability, short_seq_prob, batch_size=batch_size ) else: training, max_predictions_per_seq = (kwargs['training'], kwargs['max_predictions_per_seq']) - data_layer = nemo_nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingPreprocessedDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertPretrainingPreprocessedDataLayer( data_file, max_predictions_per_seq, batch_size=batch_size, training=training ) steps_per_epoch = math.ceil(len(data_layer) / (batch_size * args.num_gpus * batches_per_step)) - (input_ids, input_type_ids, input_mask, output_ids, output_mask, nsp_labels) = data_layer() - hidden_states = bert_model(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + input_data = data_layer() + hidden_states = bert_model( + input_ids=input_data.input_ids, token_type_ids=input_data.input_type_ids, attention_mask=input_data.input_mask + ) mlm_logits = mlm_classifier(hidden_states=hidden_states) - mlm_loss = mlm_loss_fn(logits=mlm_logits, output_ids=output_ids, output_mask=output_mask) + mlm_loss = mlm_loss_fn(logits=mlm_logits, output_ids=input_data.output_ids, output_mask=input_data.output_mask) if not args.only_mlm_loss: nsp_logits = nsp_classifier(hidden_states=hidden_states) - nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=nsp_labels) + nsp_loss = nsp_loss_fn(logits=nsp_logits, labels=input_data.labels) loss = bert_loss(loss_1=mlm_loss, loss_2=nsp_loss) else: loss = mlm_loss diff --git a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py index f8467eff6d51..8cda90810521 100644 --- a/examples/nlp/neural_machine_translation/machine_translation_tutorial.py +++ b/examples/nlp/neural_machine_translation/machine_translation_tutorial.py @@ -146,7 +146,7 @@ max_seq_length=args.max_seq_length, ) -log_softmax = nemo_nlp.nm.trainables.token_classification_nm.TokenClassifier( +log_softmax = nemo_nlp.nm.trainables.TokenClassifier( args.d_model, num_classes=tgt_tokenizer.vocab_size, num_layers=1, log_softmax=True ) diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 17bccbf5cac6..630bd46939f7 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -216,7 +216,7 @@ def create_pipeline( batches_per_step=1, mode="train", ): - data_layer = nemo_nlp.nm.data_layers.qa_squad_datalayer.BertQuestionAnsweringDataLayer( + data_layer = nemo_nlp.nm.data_layers.BertQuestionAnsweringDataLayer( mode=mode, version_2_with_negative=version_2_with_negative, batch_size=batch_size, @@ -316,7 +316,7 @@ def create_pipeline( if args.tokenizer == "sentencepiece": try: - tokenizer = nemo_nlp.SentencePieceTokenizer(model_path=args.tokenizer_model) + tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path=args.tokenizer_model) except Exception: raise ValueError( "Using --tokenizer=sentencepiece \ @@ -354,13 +354,13 @@ def create_pipeline( else: """ Use this if you're using a standard BERT model. To see the list of pretrained models, call: - nemo_nlp.huggingface.BERT.list_pretrained_models() + nemo_nlp.nm.trainables.huggingface.BERT.list_pretrained_models() """ model = model_cls(pretrained_model_name=args.pretrained_model_name) hidden_size = model.hidden_size - qa_head = nemo_nlp.nm.trainables.token_classification_nm.TokenClassifier( + qa_head = nemo_nlp.nm.trainables.TokenClassifier( hidden_size=hidden_size, num_classes=2, num_layers=1, log_softmax=False ) squad_loss = nemo_nlp.nm.losses.QuestionAnsweringLoss() From c34aa733954337f398acd6d940868be3e6d26ecc Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 12:51:20 -0800 Subject: [PATCH 51/70] fix style Signed-off-by: Oleksii Kuchaiev --- .../pytorch/torchvision/data/image_folder.py | 14 ++-- nemo/core/neural_factory.py | 28 ++++--- nemo/core/neural_modules.py | 1 - nemo/core/neural_types/elements.py | 2 + tests/asr/test_zeroDS.py | 3 +- tests/core/test_neural_modules.py | 5 +- tests/core/test_neural_modules_pytorch.py | 8 +- tests/core/test_neural_types.py | 77 +++++++++++-------- 8 files changed, 83 insertions(+), 55 deletions(-) diff --git a/nemo/backends/pytorch/torchvision/data/image_folder.py b/nemo/backends/pytorch/torchvision/data/image_folder.py index b775efb1a8f5..5c4946b5cdd5 100644 --- a/nemo/backends/pytorch/torchvision/data/image_folder.py +++ b/nemo/backends/pytorch/torchvision/data/image_folder.py @@ -27,12 +27,14 @@ def output_ports(self): 0: AxisType(BatchTag) """ return { - "image": NeuralType({ - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, self._input_size), - 3: AxisType(WidthTag, self._input_size), - }), + "image": NeuralType( + { + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, self._input_size), + 3: AxisType(WidthTag, self._input_size), + } + ), "label": NeuralType({0: AxisType(BatchTag)}), } diff --git a/nemo/core/neural_factory.py b/nemo/core/neural_factory.py index 7a7a5154ef2c..0692ea46095c 100644 --- a/nemo/core/neural_factory.py +++ b/nemo/core/neural_factory.py @@ -463,12 +463,14 @@ def __get_pytorch_module(self, name, collection, params, pretrained): _nm_name = name.lower() if _nm_name == "resnet18": input_ports = { - "x": NeuralType({ - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - }) + "x": NeuralType( + { + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, 224), + 3: AxisType(WidthTag, 224), + } + ) } output_ports = {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} @@ -481,12 +483,14 @@ def __get_pytorch_module(self, name, collection, params, pretrained): ) elif _nm_name == "resnet50": input_ports = { - "x": NeuralType({ - 0: AxisType(BatchTag), - 1: AxisType(ChannelTag), - 2: AxisType(HeightTag, 224), - 3: AxisType(WidthTag, 224), - }) + "x": NeuralType( + { + 0: AxisType(BatchTag), + 1: AxisType(ChannelTag), + 2: AxisType(HeightTag, 224), + 3: AxisType(WidthTag, 224), + } + ) } output_ports = {"output": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)})} diff --git a/nemo/core/neural_modules.py b/nemo/core/neural_modules.py index 74bf9cb6108d..25e42c7824fa 100644 --- a/nemo/core/neural_modules.py +++ b/nemo/core/neural_modules.py @@ -311,7 +311,6 @@ def __call__(self, **kwargs): return result - def __str__(self): return self.__class__.__name__ diff --git a/nemo/core/neural_types/elements.py b/nemo/core/neural_types/elements.py index 59f818ee2688..5d410b90ebde 100644 --- a/nemo/core/neural_types/elements.py +++ b/nemo/core/neural_types/elements.py @@ -102,6 +102,7 @@ class VoidType(ElementType): It is a good practice to use this type only as necessary. For example, when you need template-like functionality. """ + def compare(cls, second: abc.ABCMeta) -> NeuralTypeComparisonResult: return NeuralTypeComparisonResult.SAME @@ -152,6 +153,7 @@ class AudioSignal(ElementType): freq (int): sampling frequency of a signal. Note that two signals will only be the same if their freq is the same. """ + def __init__(self, freq: int = 16000): self._params = {} self._params['freq'] = freq diff --git a/tests/asr/test_zeroDS.py b/tests/asr/test_zeroDS.py index 6dc9926a597d..a413e1f2e514 100644 --- a/tests/asr/test_zeroDS.py +++ b/tests/asr/test_zeroDS.py @@ -109,7 +109,8 @@ def test_asr_with_zero_ds(self): # "transcript_length": NeuralType({0: AxisType(BatchTag)}), "processed_signal": NeuralType( (AxisType(AxisKind.Batch), AxisType(AxisKind.Dimension, 64), AxisType(AxisKind.Time, 64)), - SpectrogramType()), + SpectrogramType(), + ), "processed_length": NeuralType(tuple('B'), LengthsType()), "transcript": NeuralType((AxisType(AxisKind.Batch), AxisType(AxisKind.Time, 64)), LabelsType()), "transcript_length": NeuralType(tuple('B'), LengthsType()), diff --git a/tests/core/test_neural_modules.py b/tests/core/test_neural_modules.py index 5484285b8e50..04e82c2802bf 100644 --- a/tests/core/test_neural_modules.py +++ b/tests/core/test_neural_modules.py @@ -23,8 +23,9 @@ class NeuralModulesTests(NeMoUnitTest): def test_call_TaylorNet(self): - x_tg = nemo.core.neural_modules.NmTensor(producer=None, producer_args=None, name=None, ntype=NeuralType(( - 'B', 'D'), ChannelType())) + x_tg = nemo.core.neural_modules.NmTensor( + producer=None, producer_args=None, name=None, ntype=NeuralType(('B', 'D'), ChannelType()) + ) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) # note that real port's name: x was used diff --git a/tests/core/test_neural_modules_pytorch.py b/tests/core/test_neural_modules_pytorch.py index 8f43f2d7356f..d0cfbc44c62b 100644 --- a/tests/core/test_neural_modules_pytorch.py +++ b/tests/core/test_neural_modules_pytorch.py @@ -69,8 +69,12 @@ def test_constructor_TaylorNet(self): self.assertEqual(tn.init_params["dim"], 4) def test_call_TaylorNet(self): - x_tg = nemo.core.neural_modules.NmTensor(producer=None, producer_args=None, name=None, ntype=NeuralType( - elements_type=ChannelType(), axes=('B', 'D'))) + x_tg = nemo.core.neural_modules.NmTensor( + producer=None, + producer_args=None, + name=None, + ntype=NeuralType(elements_type=ChannelType(), axes=('B', 'D')), + ) tn = nemo.backends.pytorch.tutorials.TaylorNet(dim=4) # note that real port's name: x was used diff --git a/tests/core/test_neural_types.py b/tests/core/test_neural_types.py index c82740c6a712..e31fd08941d3 100644 --- a/tests/core/test_neural_types.py +++ b/tests/core/test_neural_types.py @@ -37,7 +37,8 @@ class NeuralTypeSystemTests(NeMoUnitTest): def test_short_vs_long_version(self): long_version = NeuralType( axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)), - elements_type=AcousticEncodedRepresentation()) + elements_type=AcousticEncodedRepresentation(), + ) short_version = NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()) self.assertEqual(long_version.compare(short_version), NeuralTypeComparisonResult.SAME) self.assertEqual(short_version.compare(long_version), NeuralTypeComparisonResult.SAME) @@ -81,20 +82,26 @@ def test_singletone(self): self.assertEqual(loss_output2.compare(loss_output1), NeuralTypeComparisonResult.SAME) def test_list_of_lists(self): - T1 = NeuralType(axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=True), - AxisType(kind=AxisKind.Time, size=None, is_list=True), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), elements_type=ChannelType()) - T2 = NeuralType(axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=False), - AxisType(kind=AxisKind.Time, size=None, is_list=False), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), elements_type=ChannelType()) + T1 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + T2 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) # TODO: should this be incompatible instead??? self.assertEqual(T1.compare(T2), NeuralTypeComparisonResult.TRANSPOSE_SAME) @@ -112,20 +119,26 @@ def test_big_void(self): btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) - t1 = NeuralType(axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=True), - AxisType(kind=AxisKind.Time, size=None, is_list=True), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), elements_type=ChannelType()) - t2 = NeuralType(axes=( - AxisType(kind=AxisKind.Batch, size=None, is_list=False), - AxisType(kind=AxisKind.Time, size=None, is_list=False), - AxisType(kind=AxisKind.Dimension, size=32, is_list=False), - AxisType(kind=AxisKind.Dimension, size=128, is_list=False), - AxisType(kind=AxisKind.Dimension, size=256, is_list=False), - ), elements_type=ChannelType()) + t1 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + t2 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) self.assertEqual(big_void_1.compare(btc_spctr), NeuralTypeComparisonResult.SAME) self.assertEqual(big_void_1.compare(btc_spct_bad), NeuralTypeComparisonResult.SAME) @@ -156,8 +169,10 @@ def wrong(): self.assertRaises(NeuralPortNmTensorMismatchError, wrong) def test_unspecified_dimensions(self): - t0 = NeuralType((AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), - SpectrogramType()) + t0 = NeuralType( + (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), + SpectrogramType(), + ) t1 = NeuralType(('B', 'T', 'C'), SpectrogramType()) self.assertEqual(t1.compare(t0), NeuralTypeComparisonResult.SAME) self.assertEqual(t0.compare(t1), NeuralTypeComparisonResult.DIM_INCOMPATIBLE) From b69269296ee79c292c5de0f0b59b730794f619f2 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 13:17:38 -0800 Subject: [PATCH 52/70] fix trade example Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/data_layers/state_tracking_trade_datalayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py index 8435dc976b8c..2b7e3800928a 100644 --- a/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py +++ b/nemo/collections/nlp/nm/data_layers/state_tracking_trade_datalayer.py @@ -76,7 +76,7 @@ def output_ports(self): # "turn_domain": NeuralType(None), "src_ids": NeuralType(('B', 'T'), ChannelType()), "src_lens": NeuralType(tuple('B'), LengthsType()), - "tgt_ids": NeuralType(('B', 'D', 'T'), ChannelType()), + "tgt_ids": NeuralType(('B', 'D', 'T'), LabelsType()), "tgt_lens": NeuralType(('B', 'D'), LengthsType()), "gating_labels": NeuralType(('B', 'D'), LabelsType()), "turn_domain": NeuralType(), From 48f64a6efc72e9be1d9606c29776ee58b0d239e9 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 13:53:56 -0800 Subject: [PATCH 53/70] incorporate PR feedback Signed-off-by: Yang Zhang --- docs/sources/source/nlp/bert_pretraining.rst | 17 ++--- .../sources/source/nlp/question_answering.rst | 20 +----- .../glue_benchmark_with_bert.py | 8 +-- .../nlp/language_modeling/bert_pretraining.py | 9 +-- .../question_answering_squad.py | 62 +++---------------- .../punctuation_capitalization.py | 9 +-- .../token_classification.py | 9 +-- .../nlp/data/datasets/lm_bert_dataset.py | 2 +- .../nlp/data/datasets/qa_squad_dataset.py | 4 +- .../tokenizers/sentencepiece_tokenizer.py | 2 +- nemo/collections/nlp/utils/__init__.py | 1 + tests/nlp/test_spc_tokenizer.py | 60 +++--------------- 12 files changed, 32 insertions(+), 171 deletions(-) diff --git a/docs/sources/source/nlp/bert_pretraining.rst b/docs/sources/source/nlp/bert_pretraining.rst index 2e6793874212..26946ea836e8 100644 --- a/docs/sources/source/nlp/bert_pretraining.rst +++ b/docs/sources/source/nlp/bert_pretraining.rst @@ -61,7 +61,7 @@ If have an available vocab, say the ``vocab.txt`` file from any `pretrained BERT .. code-block:: python - data_desc = BERTPretrainingDataDesc(args.dataset_name, + data_desc = nemo_nlp.data.BERTPretrainingDataDesc(args.dataset_name, args.data_dir, args.vocab_size, args.sample_size, @@ -76,21 +76,14 @@ To train on a Chinese dataset, you should use `NemoBertTokenizer`. .. code-block:: python # If you're using a custom vocabulary, create your tokenizer like this - tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path="tokenizer.model") + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this - tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + tokenizer = nemo_nlp.data.NemoBertTokenizer(vocab_file="vocab.txt") # or - tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") + tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model="bert-base-uncased") Create the model ---------------- diff --git a/docs/sources/source/nlp/question_answering.rst b/docs/sources/source/nlp/question_answering.rst index 56c91645ed2a..80482c875c97 100644 --- a/docs/sources/source/nlp/question_answering.rst +++ b/docs/sources/source/nlp/question_answering.rst @@ -68,26 +68,10 @@ This will tokenize text following the mapping of the original BERT model. hidden_size = model.hidden_size tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='bert', pretrained_model="bert-base-uncased") # to use RoBERTa tokenizer, run e.g. - special_tokens_roberta = { - "unk_token": "", - "sep_token": "", - "pad_token": "", - "bos_token": "", - "mask_token": "", - "eos_token": "", - "cls_token": "", - } + special_tokens_roberta = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['roberta'] tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='roberta', pretrained_model="roberta-base", special_tokens=special_tokens_roberta) # to use Albert tokenizer, run e.g. - special_tokens_albert = { - "unk_token": "", - "sep_token": "[SEP]", - "eos_token": "[SEP]", - "pad_token": "", - "cls_token": "[CLS]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - } + special_tokens_albert = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['albert'] tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='albert', pretrained_model="albert-base-v1", special_tokens=special_tokens_albert) diff --git a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py index a6aa20380b9d..6c23618a7329 100644 --- a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py +++ b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py @@ -199,13 +199,7 @@ Replace BERT-STEP-150000.pt with the path to your checkpoint. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 0273615d2b38..eaf40dc454ed 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -160,14 +160,7 @@ args.max_seq_length = config['max_position_embeddings'] if not args.preprocessed_data: - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] data_desc = BERTPretrainingDataDesc( args.dataset_name, args.data_dir, diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 630bd46939f7..415a04b23785 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -248,50 +248,12 @@ def create_pipeline( MODEL_CLASSES = { - "bert": { - "model_name": "bert-base-uncased", - "tokenizer_name": "bert-base-uncased", - "model": nemo_nlp.nm.trainables.huggingface.BERT, - "special_tokens": { - "unk_token": "[UNK]", - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - }, - }, - "roberta": { - "model_name": "roberta-base", - "tokenizer_name": "roberta-base", - "model": nemo_nlp.nm.trainables.huggingface.Roberta, - "special_tokens": { - "unk_token": "", - "sep_token": "", - "pad_token": "", - "bos_token": "", - "mask_token": "", - "eos_token": "", - "cls_token": "", - }, - }, - "albert": { - "model_name": "albert-base-v2", - "tokenizer_name": "albert-base-v2", - "model": nemo_nlp.nm.trainables.huggingface.Albert, - "special_tokens": { - "unk_token": "", - "sep_token": "[SEP]", - "eos_token": "[SEP]", - "pad_token": "", - "cls_token": "[CLS]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - }, - }, + 'bert': nemo_nlp.nm.trainables.huggingface.BERT, + 'albert': nemo_nlp.nm.trainables.huggingface.Albert, + 'roberta': nemo_nlp.nm.trainables.huggingface.Roberta, } + if __name__ == "__main__": args = parse_args() if not os.path.exists(args.dev_file): @@ -322,20 +284,14 @@ def create_pipeline( "Using --tokenizer=sentencepiece \ requires valid --tokenizer_model" ) - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS[args.model_type] tokenizer.add_special_tokens(special_tokens) else: tokenizer_cls = nemo_nlp.data.NemoBertTokenizer - tokenizer_special_tokens = MODEL_CLASSES[args.model_type]["special_tokens"] - model_cls = MODEL_CLASSES[args.model_type]["model"] - model_name = MODEL_CLASSES[args.model_type]["model_name"] - tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] + tokenizer_special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS[args.model_type] + model_cls = MODEL_CLASSES[args.model_type] + model_name = nemo_nlp.utils.MODEL_NAMES[args.model_type]["model_name"] + tokenizer_name = nemo_nlp.utils.MODEL_NAMES[args.model_type]["tokenizer_name"] if args.pretrained_model_name is None: args.pretrained_model_name = model_name diff --git a/examples/nlp/token_classification/punctuation_capitalization.py b/examples/nlp/token_classification/punctuation_capitalization.py index 755bf64c9600..b74eeff89663 100644 --- a/examples/nlp/token_classification/punctuation_capitalization.py +++ b/examples/nlp/token_classification/punctuation_capitalization.py @@ -134,14 +134,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/examples/nlp/token_classification/token_classification.py b/examples/nlp/token_classification/token_classification.py index 265e43efe7be..7254929863f1 100644 --- a/examples/nlp/token_classification/token_classification.py +++ b/examples/nlp/token_classification/token_classification.py @@ -121,14 +121,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py index b542d08adcce..61b74f933c60 100644 --- a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py +++ b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py @@ -141,7 +141,7 @@ def __len__(self): def __getitem__(self, idx, min_doc_length=16): # Each sequence has three special tokens, as follows: - # [CLS] [SEP] [SEP] + # tokenizer.cls_token tokenizer.sep_token tokenizer.eos_token num_special_tokens = 3 max_num_tokens = self.max_seq_length - num_special_tokens diff --git a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py index 86e2a9f4060b..b927f83ead38 100644 --- a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py +++ b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py @@ -448,7 +448,7 @@ def convert_examples_to_features( all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text ) - # The -3 accounts for [CLS], [SEP] and [SEP] + # The -3 accounts for tokenizer.cls_token, tokenizer.sep_token and tokenizer.eos_token # doc_spans contains all possible contexts options of given length max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) @@ -506,7 +506,7 @@ def convert_examples_to_features( # calculate start and end position in final array # of tokens in answer if no answer, - # 0 for both pointing to [CLS] + # 0 for both pointing to tokenizer.cls_token start_position = None end_position = None if has_groundtruth and not example.is_impossible: diff --git a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py index 8405682854e5..0eaf8873898b 100644 --- a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py @@ -25,7 +25,7 @@ class SentencePieceTokenizer(TokenizerSpec): def __init__(self, model_path, special_tokens={}): self.tokenizer = spm.SentencePieceProcessor() self.tokenizer.Load(model_path) - # wihtout special tokens + # without special tokens self.original_vocab_size = self.tokenizer.get_piece_size() self.vocab_size = self.tokenizer.get_piece_size() self.special_token_to_id = {} diff --git a/nemo/collections/nlp/utils/__init__.py b/nemo/collections/nlp/utils/__init__.py index 49948c01f0c6..9a0f97ecdc63 100644 --- a/nemo/collections/nlp/utils/__init__.py +++ b/nemo/collections/nlp/utils/__init__.py @@ -1,3 +1,4 @@ from nemo.collections.nlp.utils.callback_utils import * from nemo.collections.nlp.utils.common_nlp_utils import * +from nemo.collections.nlp.utils.huggingface_utils import * from nemo.collections.nlp.utils.loss_utils import * diff --git a/tests/nlp/test_spc_tokenizer.py b/tests/nlp/test_spc_tokenizer.py index dcae02027eb0..e291ce267d92 100644 --- a/tests/nlp/test_spc_tokenizer.py +++ b/tests/nlp/test_spc_tokenizer.py @@ -16,6 +16,7 @@ # limitations under the License. # ============================================================================= +import nemo.collections.nlp as nemo_nlp from nemo.collections.nlp.data import SentencePieceTokenizer from tests.common_setup import NeMoUnitTest @@ -23,28 +24,13 @@ class TestSPCTokenizer(NeMoUnitTest): def test_add_special_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) self.assertTrue(tokenizer.vocab_size == tokenizer.original_vocab_size + len(set(special_tokens.values()))) def test_text_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -66,15 +52,7 @@ def test_tokens_to_text(self): def test_text_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -87,15 +65,7 @@ def test_text_to_ids(self): def test_ids_to_text(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -106,15 +76,7 @@ def test_ids_to_text(self): def test_tokens_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -128,15 +90,7 @@ def test_tokens_to_ids(self): def test_ids_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" From 46c9755302df8ec9e5f4ca0e417bd9a51648993b Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 13:53:56 -0800 Subject: [PATCH 54/70] incorporate PR feedback Signed-off-by: Yang Zhang --- docs/sources/source/nlp/bert_pretraining.rst | 17 ++--- .../sources/source/nlp/question_answering.rst | 20 +---- .../glue_benchmark_with_bert.py | 8 +- .../nlp/language_modeling/bert_pretraining.py | 9 +-- .../question_answering_squad.py | 76 ++++--------------- .../punctuation_capitalization.py | 9 +-- .../token_classification.py | 9 +-- .../nlp/data/datasets/lm_bert_dataset.py | 2 +- .../nlp/data/datasets/qa_squad_dataset.py | 4 +- .../tokenizers/sentencepiece_tokenizer.py | 2 +- nemo/collections/nlp/utils/__init__.py | 1 + tests/nlp/test_spc_tokenizer.py | 60 ++------------- 12 files changed, 39 insertions(+), 178 deletions(-) diff --git a/docs/sources/source/nlp/bert_pretraining.rst b/docs/sources/source/nlp/bert_pretraining.rst index 2e6793874212..26946ea836e8 100644 --- a/docs/sources/source/nlp/bert_pretraining.rst +++ b/docs/sources/source/nlp/bert_pretraining.rst @@ -61,7 +61,7 @@ If have an available vocab, say the ``vocab.txt`` file from any `pretrained BERT .. code-block:: python - data_desc = BERTPretrainingDataDesc(args.dataset_name, + data_desc = nemo_nlp.data.BERTPretrainingDataDesc(args.dataset_name, args.data_dir, args.vocab_size, args.sample_size, @@ -76,21 +76,14 @@ To train on a Chinese dataset, you should use `NemoBertTokenizer`. .. code-block:: python # If you're using a custom vocabulary, create your tokenizer like this - tokenizer = SentencePieceTokenizer(model_path="tokenizer.model") - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + tokenizer = nemo_nlp.data.SentencePieceTokenizer(model_path="tokenizer.model") + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) # Otherwise, create your tokenizer like this - tokenizer = NemoBertTokenizer(vocab_file="vocab.txt") + tokenizer = nemo_nlp.data.NemoBertTokenizer(vocab_file="vocab.txt") # or - tokenizer = NemoBertTokenizer(pretrained_model="bert-base-uncased") + tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_model="bert-base-uncased") Create the model ---------------- diff --git a/docs/sources/source/nlp/question_answering.rst b/docs/sources/source/nlp/question_answering.rst index 56c91645ed2a..80482c875c97 100644 --- a/docs/sources/source/nlp/question_answering.rst +++ b/docs/sources/source/nlp/question_answering.rst @@ -68,26 +68,10 @@ This will tokenize text following the mapping of the original BERT model. hidden_size = model.hidden_size tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='bert', pretrained_model="bert-base-uncased") # to use RoBERTa tokenizer, run e.g. - special_tokens_roberta = { - "unk_token": "", - "sep_token": "", - "pad_token": "", - "bos_token": "", - "mask_token": "", - "eos_token": "", - "cls_token": "", - } + special_tokens_roberta = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['roberta'] tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='roberta', pretrained_model="roberta-base", special_tokens=special_tokens_roberta) # to use Albert tokenizer, run e.g. - special_tokens_albert = { - "unk_token": "", - "sep_token": "[SEP]", - "eos_token": "[SEP]", - "pad_token": "", - "cls_token": "[CLS]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - } + special_tokens_albert = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['albert'] tokenizer = nemo_nlp.data.NemoBertTokenizer(bert_derivate='albert', pretrained_model="albert-base-v1", special_tokens=special_tokens_albert) diff --git a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py index a6aa20380b9d..6c23618a7329 100644 --- a/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py +++ b/examples/nlp/glue_benchmark/glue_benchmark_with_bert.py @@ -199,13 +199,7 @@ Replace BERT-STEP-150000.pt with the path to your checkpoint. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 0273615d2b38..eaf40dc454ed 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -160,14 +160,7 @@ args.max_seq_length = config['max_position_embeddings'] if not args.preprocessed_data: - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] data_desc = BERTPretrainingDataDesc( args.dataset_name, args.data_dir, diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py index 630bd46939f7..6f7197e349aa 100755 --- a/examples/nlp/question_answering/question_answering_squad.py +++ b/examples/nlp/question_answering/question_answering_squad.py @@ -248,50 +248,12 @@ def create_pipeline( MODEL_CLASSES = { - "bert": { - "model_name": "bert-base-uncased", - "tokenizer_name": "bert-base-uncased", - "model": nemo_nlp.nm.trainables.huggingface.BERT, - "special_tokens": { - "unk_token": "[UNK]", - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - }, - }, - "roberta": { - "model_name": "roberta-base", - "tokenizer_name": "roberta-base", - "model": nemo_nlp.nm.trainables.huggingface.Roberta, - "special_tokens": { - "unk_token": "", - "sep_token": "", - "pad_token": "", - "bos_token": "", - "mask_token": "", - "eos_token": "", - "cls_token": "", - }, - }, - "albert": { - "model_name": "albert-base-v2", - "tokenizer_name": "albert-base-v2", - "model": nemo_nlp.nm.trainables.huggingface.Albert, - "special_tokens": { - "unk_token": "", - "sep_token": "[SEP]", - "eos_token": "[SEP]", - "pad_token": "", - "cls_token": "[CLS]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - }, - }, + 'bert': nemo_nlp.nm.trainables.huggingface.BERT, + 'albert': nemo_nlp.nm.trainables.huggingface.Albert, + 'roberta': nemo_nlp.nm.trainables.huggingface.Roberta, } + if __name__ == "__main__": args = parse_args() if not os.path.exists(args.dev_file): @@ -322,31 +284,25 @@ def create_pipeline( "Using --tokenizer=sentencepiece \ requires valid --tokenizer_model" ) - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS[args.model_type] tokenizer.add_special_tokens(special_tokens) else: tokenizer_cls = nemo_nlp.data.NemoBertTokenizer - tokenizer_special_tokens = MODEL_CLASSES[args.model_type]["special_tokens"] - model_cls = MODEL_CLASSES[args.model_type]["model"] - model_name = MODEL_CLASSES[args.model_type]["model_name"] - tokenizer_name = MODEL_CLASSES[args.model_type]["tokenizer_name"] + tokenizer_special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS[args.model_type] + tokenizer_name = nemo_nlp.utils.MODEL_NAMES[args.model_type]["tokenizer_name"] + tokenizer = tokenizer_cls( + do_lower_case=args.do_lower_case, + pretrained_model=tokenizer_name, + special_tokens=tokenizer_special_tokens, + bert_derivate=args.model_type, + ) + + model_cls = MODEL_CLASSES[args.model_type] + model_name = nemo_nlp.utils.MODEL_NAMES[args.model_type]["model_name"] if args.pretrained_model_name is None: args.pretrained_model_name = model_name - tokenizer = tokenizer_cls( - do_lower_case=args.do_lower_case, - pretrained_model=tokenizer_name, - special_tokens=tokenizer_special_tokens, - bert_derivate=args.model_type, - ) - if args.bert_config is not None: with open(args.bert_config) as json_file: config = json.load(json_file) diff --git a/examples/nlp/token_classification/punctuation_capitalization.py b/examples/nlp/token_classification/punctuation_capitalization.py index 755bf64c9600..b74eeff89663 100644 --- a/examples/nlp/token_classification/punctuation_capitalization.py +++ b/examples/nlp/token_classification/punctuation_capitalization.py @@ -134,14 +134,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model, special_tokens=special_tokens) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/examples/nlp/token_classification/token_classification.py b/examples/nlp/token_classification/token_classification.py index 265e43efe7be..7254929863f1 100644 --- a/examples/nlp/token_classification/token_classification.py +++ b/examples/nlp/token_classification/token_classification.py @@ -121,14 +121,7 @@ """ Use this if you're using a BERT model that you pre-trained yourself. """ if args.tokenizer == "sentencepiece": - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer = SentencePieceTokenizer(model_path=args.tokenizer_model) elif args.tokenizer == "nemobert": tokenizer = NemoBertTokenizer(args.pretrained_bert_model) diff --git a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py index b542d08adcce..61b74f933c60 100644 --- a/nemo/collections/nlp/data/datasets/lm_bert_dataset.py +++ b/nemo/collections/nlp/data/datasets/lm_bert_dataset.py @@ -141,7 +141,7 @@ def __len__(self): def __getitem__(self, idx, min_doc_length=16): # Each sequence has three special tokens, as follows: - # [CLS] [SEP] [SEP] + # tokenizer.cls_token tokenizer.sep_token tokenizer.eos_token num_special_tokens = 3 max_num_tokens = self.max_seq_length - num_special_tokens diff --git a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py index 86e2a9f4060b..b927f83ead38 100644 --- a/nemo/collections/nlp/data/datasets/qa_squad_dataset.py +++ b/nemo/collections/nlp/data/datasets/qa_squad_dataset.py @@ -448,7 +448,7 @@ def convert_examples_to_features( all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text ) - # The -3 accounts for [CLS], [SEP] and [SEP] + # The -3 accounts for tokenizer.cls_token, tokenizer.sep_token and tokenizer.eos_token # doc_spans contains all possible contexts options of given length max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) @@ -506,7 +506,7 @@ def convert_examples_to_features( # calculate start and end position in final array # of tokens in answer if no answer, - # 0 for both pointing to [CLS] + # 0 for both pointing to tokenizer.cls_token start_position = None end_position = None if has_groundtruth and not example.is_impossible: diff --git a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py index 8405682854e5..0eaf8873898b 100644 --- a/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/nlp/data/tokenizers/sentencepiece_tokenizer.py @@ -25,7 +25,7 @@ class SentencePieceTokenizer(TokenizerSpec): def __init__(self, model_path, special_tokens={}): self.tokenizer = spm.SentencePieceProcessor() self.tokenizer.Load(model_path) - # wihtout special tokens + # without special tokens self.original_vocab_size = self.tokenizer.get_piece_size() self.vocab_size = self.tokenizer.get_piece_size() self.special_token_to_id = {} diff --git a/nemo/collections/nlp/utils/__init__.py b/nemo/collections/nlp/utils/__init__.py index 49948c01f0c6..9a0f97ecdc63 100644 --- a/nemo/collections/nlp/utils/__init__.py +++ b/nemo/collections/nlp/utils/__init__.py @@ -1,3 +1,4 @@ from nemo.collections.nlp.utils.callback_utils import * from nemo.collections.nlp.utils.common_nlp_utils import * +from nemo.collections.nlp.utils.huggingface_utils import * from nemo.collections.nlp.utils.loss_utils import * diff --git a/tests/nlp/test_spc_tokenizer.py b/tests/nlp/test_spc_tokenizer.py index dcae02027eb0..e291ce267d92 100644 --- a/tests/nlp/test_spc_tokenizer.py +++ b/tests/nlp/test_spc_tokenizer.py @@ -16,6 +16,7 @@ # limitations under the License. # ============================================================================= +import nemo.collections.nlp as nemo_nlp from nemo.collections.nlp.data import SentencePieceTokenizer from tests.common_setup import NeMoUnitTest @@ -23,28 +24,13 @@ class TestSPCTokenizer(NeMoUnitTest): def test_add_special_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) self.assertTrue(tokenizer.vocab_size == tokenizer.original_vocab_size + len(set(special_tokens.values()))) def test_text_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -66,15 +52,7 @@ def test_tokens_to_text(self): def test_text_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -87,15 +65,7 @@ def test_text_to_ids(self): def test_ids_to_text(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -106,15 +76,7 @@ def test_ids_to_text(self): def test_tokens_to_ids(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" @@ -128,15 +90,7 @@ def test_tokens_to_ids(self): def test_ids_to_tokens(self): tokenizer = SentencePieceTokenizer("./tests/data/m_common.model") - - special_tokens = { - "sep_token": "[SEP]", - "pad_token": "[PAD]", - "bos_token": "[CLS]", - "mask_token": "[MASK]", - "eos_token": "[SEP]", - "cls_token": "[CLS]", - } + special_tokens = nemo_nlp.utils.MODEL_SPECIAL_TOKENS['bert'] tokenizer.add_special_tokens(special_tokens) text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" From 0a07bb3774605225cf582aa561a7f304f937c19b Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 12 Feb 2020 14:02:57 -0800 Subject: [PATCH 55/70] add utils file Signed-off-by: Yang Zhang --- .../nlp/utils/huggingface_utils.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 nemo/collections/nlp/utils/huggingface_utils.py diff --git a/nemo/collections/nlp/utils/huggingface_utils.py b/nemo/collections/nlp/utils/huggingface_utils.py new file mode 100644 index 000000000000..98f3df9c36b7 --- /dev/null +++ b/nemo/collections/nlp/utils/huggingface_utils.py @@ -0,0 +1,54 @@ +# ============================================================================= +# Copyright 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +__all__ = ['MODEL_SPECIAL_TOKENS', 'MODEL_NAMES'] + +MODEL_SPECIAL_TOKENS = { + "bert": { + "unk_token": "[UNK]", + "sep_token": "[SEP]", + "pad_token": "[PAD]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + "eos_token": "[SEP]", + "cls_token": "[CLS]", + }, + "roberta": { + "unk_token": "", + "sep_token": "", + "pad_token": "", + "bos_token": "", + "mask_token": "", + "eos_token": "", + "cls_token": "", + }, + "albert": { + "unk_token": "", + "sep_token": "[SEP]", + "eos_token": "[SEP]", + "pad_token": "", + "cls_token": "[CLS]", + "bos_token": "[CLS]", + "mask_token": "[MASK]", + }, +} + + +MODEL_NAMES = { + "bert": {"model_name": "bert-base-uncased", "tokenizer_name": "bert-base-uncased",}, + "roberta": {"model_name": "roberta-base", "tokenizer_name": "roberta-base",}, + "albert": {"model_name": "albert-base-v2", "tokenizer_name": "albert-base-v2",}, +} From 196a24891e355c52140621f0e03c53fad9e7855d Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 14:25:13 -0800 Subject: [PATCH 56/70] fix trade example Signed-off-by: Oleksii Kuchaiev --- nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py index ea065494e8ee..aa67439b9262 100644 --- a/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py +++ b/nemo/collections/nlp/nm/losses/state_tracking_trade_loss.py @@ -74,7 +74,7 @@ def input_ports(self): # "targets": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag), 2: AxisType(TimeTag)}), # "loss_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(ChannelTag)}), "logits": NeuralType(('B', 'T', 'D', 'D'), LogitsType()), - "targets": NeuralType(('B', 'D', 'T'), ChannelType()), + "targets": NeuralType(('B', 'D', 'T'), LabelsType()), "loss_mask": NeuralType(('B', 'D'), LengthsType()), } From 48a0e360c3dcfc01fad3769acc6beec64d0eebb4 Mon Sep 17 00:00:00 2001 From: Vitaly Lavrukhin Date: Wed, 12 Feb 2020 15:18:41 -0800 Subject: [PATCH 57/70] Fixed import in notebook Signed-off-by: Vitaly Lavrukhin --- examples/asr/notebooks/2_Online_ASR_Microphone_Demo.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/asr/notebooks/2_Online_ASR_Microphone_Demo.ipynb b/examples/asr/notebooks/2_Online_ASR_Microphone_Demo.ipynb index 0a4a5842f0b8..9352d5fb0102 100644 --- a/examples/asr/notebooks/2_Online_ASR_Microphone_Demo.ipynb +++ b/examples/asr/notebooks/2_Online_ASR_Microphone_Demo.ipynb @@ -28,7 +28,8 @@ "metadata": {}, "outputs": [], "source": [ - "import nemo, nemo_asr\n", + "import nemo\n", + "import nemo.collections.asr as nemo_asr\n", "from nemo.collections.asr.helpers import post_process_predictions\n", "import numpy as np\n", "import pyaudio as pa\n", @@ -211,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -387,7 +388,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.6.10" } }, "nbformat": 4, From a736faee30e8090f2e3a16750e16f0bf2905707d Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 16:41:46 -0800 Subject: [PATCH 58/70] fixing bert alternatives types Signed-off-by: Oleksii Kuchaiev --- .../common/huggingface/albert_nm.py | 38 ++++++------------- .../common/huggingface/roberta_nm.py | 38 ++++++------------- 2 files changed, 22 insertions(+), 54 deletions(-) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py index 9252e47ae2be..5c27e456b753 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import NeuralType, ChannelType __all__ = ['Albert'] @@ -54,40 +54,24 @@ class Albert(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - token_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - attention_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ + # return { + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # } return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "token_type_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), ChannelType()), } @property def output_ports(self): """Returns definitions of module output ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + #return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( self, diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py index eed3a559f09c..773c0275b9a5 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import AxisType, BatchTag, ChannelTag, NeuralType, TimeTag +from nemo.core.neural_types import NeuralType, ChannelType __all__ = ['Roberta'] @@ -54,40 +54,24 @@ class Roberta(TrainableNM): @property def input_ports(self): """Returns definitions of module input ports. - - input_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - token_type_ids: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - attention_mask: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) """ + # return { + # "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + # } return { - "input_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "token_type_ids": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), - "attention_mask": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag)}), + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "token_type_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), ChannelType()), } @property def output_ports(self): """Returns definitions of module output ports. - - hidden_states: - 0: AxisType(BatchTag) - - 1: AxisType(TimeTag) - - 2: AxisType(ChannelTag) """ - return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + #return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( self, From 48a7ede23cc5234037f74b3e7817e32e4b9e86d1 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 16:44:56 -0800 Subject: [PATCH 59/70] fix style Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/trainables/common/huggingface/albert_nm.py | 4 ++-- .../nlp/nm/trainables/common/huggingface/roberta_nm.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py index 5c27e456b753..0dc2de6e3b88 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import NeuralType, ChannelType +from nemo.core.neural_types import ChannelType, NeuralType __all__ = ['Albert'] @@ -70,7 +70,7 @@ def input_ports(self): def output_ports(self): """Returns definitions of module output ports. """ - #return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py index 773c0275b9a5..c8227023cc3e 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py @@ -25,7 +25,7 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_modules import PretrainedModelInfo -from nemo.core.neural_types import NeuralType, ChannelType +from nemo.core.neural_types import ChannelType, NeuralType __all__ = ['Roberta'] @@ -70,7 +70,7 @@ def input_ports(self): def output_ports(self): """Returns definitions of module output ports. """ - #return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} + # return {"hidden_states": NeuralType({0: AxisType(BatchTag), 1: AxisType(TimeTag), 2: AxisType(ChannelTag)})} return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} def __init__( From 2575834f210867c2f1c8362bc678d00b0eef34f6 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Wed, 12 Feb 2020 16:53:56 -0800 Subject: [PATCH 60/70] fixing headers in albert and roberta Signed-off-by: Oleksii Kuchaiev --- .../nlp/nm/trainables/common/huggingface/albert_nm.py | 2 +- .../nlp/nm/trainables/common/huggingface/roberta_nm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py index 0dc2de6e3b88..9df214302072 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/albert_nm.py @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright 2019 AI Applications Design Team at NVIDIA. All Rights Reserved. +# Copyright 2020 NVIDIA. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py index c8227023cc3e..2f0396172d3b 100644 --- a/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py +++ b/nemo/collections/nlp/nm/trainables/common/huggingface/roberta_nm.py @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright 2019 AI Applications Design Team at NVIDIA. All Rights Reserved. +# Copyright 2020 NVIDIA. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From aac6f80aa0092e69872dbda8617d3d7da1f5e91a Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 12 Feb 2020 17:10:51 -0800 Subject: [PATCH 61/70] offload cache to cpu memory by default Signed-off-by: Jason --- nemo/backends/pytorch/actions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index deec27eee087..1b28acb6b982 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -724,6 +724,8 @@ def _infer( for t in tensors_to_return: if t.unique_name in registered_e_tensors: del registered_e_tensors[t.unique_name] + else: + registered_e_tensors[t.unique_name] = t.cuda() # Need to check for device type mismatch else: if isinstance(data, torch.Tensor): @@ -751,7 +753,7 @@ def _infer( # if isinstance(tensor, torch.Tensor): # registered_e_tensors[name] = tensor.cpu() if cache: - self.append_to_cache(registered_e_tensors) + self.append_to_cache(registered_e_tensors, offload_to_cpu) # If distributed. For the outer loop, we need to ensure that # all processes loop through the elements in the same order @@ -809,7 +811,7 @@ def _infer( # For all other ranks return None - def append_to_cache(self, registered_tensors: dict): + def append_to_cache(self, registered_tensors: dict, offload_to_cpu): """Simpler helper function to add results of __nm_graph_forward_pass to current cache. """ From 9cc44736d5675ad430ba81ffea65accde1e5d397 Mon Sep 17 00:00:00 2001 From: Jocelyn Date: Wed, 12 Feb 2020 18:08:31 -0800 Subject: [PATCH 62/70] Fix calculation for number of hours of audio data (#362) * Fix calculation for number of hours of audio data Signed-off-by: Jocelyn Huang * Reformatted w/ Black Signed-off-by: Jocelyn Huang --- nemo/collections/asr/parts/dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/parts/dataset.py b/nemo/collections/asr/parts/dataset.py index 770bd9e2460f..7f8107e5c9d3 100644 --- a/nemo/collections/asr/parts/dataset.py +++ b/nemo/collections/asr/parts/dataset.py @@ -279,7 +279,8 @@ def __init__( if id2dur: # utt2dur durations are in seconds logging.info( - f"Dataset loaded with {duration / 60 : .2f} hours. " f"Filtered {filtered_duration / 60 : .2f} hours." + f"Dataset loaded with {duration / 3600 : .2f} hours. " + f"Filtered {filtered_duration / 3600 : .2f} hours." ) self.data = data From 5b728dce595ca062fd82a5ad8b982d1592666e42 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 12 Feb 2020 18:21:14 -0800 Subject: [PATCH 63/70] cache is now offloaded to cpu mem Signed-off-by: Jason --- nemo/backends/pytorch/actions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 1b28acb6b982..23fa9ceb653b 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -715,6 +715,7 @@ def _infer( loop_iterator = eval_dataloader for epoch_i, data in enumerate(loop_iterator, 0): + print(torch.cuda.memory_allocated()) if verbose and (num_batches < 10 or (epoch_i % int(num_batches / 10) == 0)): logging.info(f"Evaluating batch {epoch_i} out of {num_batches}") tensors = [] @@ -724,9 +725,9 @@ def _infer( for t in tensors_to_return: if t.unique_name in registered_e_tensors: del registered_e_tensors[t.unique_name] - else: - registered_e_tensors[t.unique_name] = t.cuda() # Need to check for device type mismatch + for t in registered_e_tensors: + registered_e_tensors[t].to(dl_device) else: if isinstance(data, torch.Tensor): data = (data,) @@ -815,6 +816,9 @@ def append_to_cache(self, registered_tensors: dict, offload_to_cpu): """Simpler helper function to add results of __nm_graph_forward_pass to current cache. """ + if offload_to_cpu: + for t in registered_tensors: + registered_tensors[t] = registered_tensors[t].cpu() self.cache.append(registered_tensors) def clear_cache(self): From 174e508732d8c1b71f6c5ec594f0c9bc34adb1b7 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 12 Feb 2020 18:21:59 -0800 Subject: [PATCH 64/70] remove print statement Signed-off-by: Jason --- nemo/backends/pytorch/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/backends/pytorch/actions.py b/nemo/backends/pytorch/actions.py index 23fa9ceb653b..0516d08d68ce 100644 --- a/nemo/backends/pytorch/actions.py +++ b/nemo/backends/pytorch/actions.py @@ -715,7 +715,7 @@ def _infer( loop_iterator = eval_dataloader for epoch_i, data in enumerate(loop_iterator, 0): - print(torch.cuda.memory_allocated()) + logging.debug(torch.cuda.memory_allocated()) if verbose and (num_batches < 10 or (epoch_i % int(num_batches / 10) == 0)): logging.info(f"Evaluating batch {epoch_i} out of {num_batches}") tensors = [] From cc6a0b06ffa563d4262aa77b567eae447737dd8c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta <56979727+tkornuta-nvidia@users.noreply.github.com> Date: Wed, 12 Feb 2020 22:39:57 -0800 Subject: [PATCH 65/70] Unjenkinsing JASPER example (#361) * unjenkinsing jasper_an4 example Signed-off-by: Tomasz Kornuta * jenkins fix - passing proper paths as args Signed-off-by: Tomasz Kornuta * dummy commit to trigger CI Signed-off-by: Tomasz Kornuta * add fix Signed-off-by: Jason * update jenkinsfile Signed-off-by: Jason * fix an4 example Signed-off-by: Jason Co-authored-by: Jason --- Jenkinsfile | 5 +++-- examples/asr/jasper_an4.py | 6 +++--- nemo/collections/asr/parts/manifest.py | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4d3c66c1ad12..7f15b83b4a75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -189,7 +189,7 @@ pipeline { } stage('Jasper AN4 O2') { steps { - sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=1 python jasper_an4.py --amp_opt_level=O2 --num_epochs=35 --test_after_training --work_dir=O2' + sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=1 python jasper_an4.py --amp_opt_level=O2 --num_epochs=35 --test_after_training --work_dir=O2 --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --eval_datasets=/home/mrjenkins/TestData/an4_dataset/an4_val.json' } } } @@ -211,11 +211,12 @@ pipeline { parallel { stage('Jasper AN4 2 GPUs') { steps { - sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 jasper_an4.py --num_epochs=40 --batch_size=24 --work_dir=multi_gpu --test_after_training' + sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 jasper_an4.py --num_epochs=40 --batch_size=24 --work_dir=multi_gpu --test_after_training --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --eval_datasets=/home/mrjenkins/TestData/an4_dataset/an4_val.json' } } } } + stage('TTS Tests') { failFast true diff --git a/examples/asr/jasper_an4.py b/examples/asr/jasper_an4.py index 1ed10e0437e5..6f2de721e13b 100644 --- a/examples/asr/jasper_an4.py +++ b/examples/asr/jasper_an4.py @@ -125,7 +125,7 @@ def main(): # Overwrite default args parser.add_argument("--train_dataset", type=str, help="training dataset path") - parser.add_argument("--eval_datasets", type=str, nargs=1, help="validation dataset path") + parser.add_argument("--eval_datasets", type=str, help="validation dataset path") # Create new args # parser.add_argument("--lm", default="./an4-lm.3gram.binary", type=str) @@ -136,8 +136,8 @@ def main(): parser.add_argument("--beta2", default=0.25, type=float) parser.set_defaults( model_config="./configs/jasper_an4.yaml", - train_dataset="/home/mrjenkins/TestData/an4_dataset/an4_train.json", - eval_datasets="/home/mrjenkins/TestData/an4_dataset/an4_val.json", + train_dataset="~/TestData/an4_dataset/an4_train.json", + eval_datasets="~/TestData/an4_dataset/an4_val.json", work_dir="./tmp", optimizer="novograd", num_epochs=50, diff --git a/nemo/collections/asr/parts/manifest.py b/nemo/collections/asr/parts/manifest.py index d91b2f80ed28..5107dbe4062e 100644 --- a/nemo/collections/asr/parts/manifest.py +++ b/nemo/collections/asr/parts/manifest.py @@ -1,5 +1,6 @@ # Copyright (c) 2019 NVIDIA Corporation import json +from os.path import expanduser from typing import Any, Dict, Iterator, List, Union @@ -42,14 +43,14 @@ def item_iter(manifests_files: Union[str, List[str]]) -> Iterator[Dict[str, Any] manifests_files = [manifests_files] for manifest_file in manifests_files: - with open(manifest_file, 'r') as f: + with open(expanduser(manifest_file), 'r') as f: for line in f: - item = __parse_item(line) + item = __parse_item(line, manifest_file) yield item -def __parse_item(line: str) -> Dict[str, Any]: +def __parse_item(line: str, manifest_file: str) -> Dict[str, Any]: item = json.loads(line) # Audio file @@ -61,6 +62,7 @@ def __parse_item(line: str) -> Dict[str, Any]: raise ValueError( f"Manifest file {manifest_file} has invalid json line " f"structure: {line} without proper audio file key." ) + item['audio_file'] = expanduser(item['audio_file']) # Duration. if 'duration' not in item: From 2410ba88be09424fcbdf4976b35f4037e6b68f48 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 13 Feb 2020 10:19:37 -0800 Subject: [PATCH 66/70] Update CI to reduce number of tests (#358) * update ci to reduce tests Signed-off-by: Jason * woah, you changed a python file, better run unittests Signed-off-by: Jason * remove changeset because it doesn't do what I want it to do Signed-off-by: Jason * style Signed-off-by: Jason * update test name Signed-off-by: Jason --- Jenkinsfile | 129 ++++++++++++++++++++++++++++++----------- examples/asr/jasper.py | 4 +- 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7f15b83b4a75..bfa626d141ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,28 +11,56 @@ pipeline { } stages { - stage('PyTorch version') { + stage('L0: PyTorch version') { steps { sh 'python -c "import torch; print(torch.__version__)"' } } - stage('Install test requirements') { + stage('L0: Install test requirements') { steps { sh 'apt-get update && apt-get install -y bc && pip install -r requirements/requirements_test.txt' } } - stage('Code formatting checks') { + stage('L0: Code formatting checks') { steps { sh 'python setup.py style' } } - stage('Unittests ALL') { + stage('L0: Unittests ALL') { steps { sh './reinstall.sh && python -m unittest' } } - stage('Parallel NLP-BERT pretraining') { + stage('L1: Parallel Stage1') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } + failFast true + parallel { + stage('Simplest test') { + steps { + sh 'cd examples/start_here && CUDA_VISIBLE_DEVICES=0 python simplest_example.py' + } + } + stage ('Chatbot test') { + steps { + sh 'cd examples/start_here && CUDA_VISIBLE_DEVICES=1 python chatbot_example.py' + } + } + } + } + + stage('L1: Parallel NLP-BERT pretraining') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true parallel { stage('BERT on the fly preprocessing') { @@ -52,25 +80,13 @@ pipeline { } } - stage('Parallel Stage1') { - failFast true - parallel { - stage('Simplest test') { - steps { - sh 'cd examples/start_here && CUDA_VISIBLE_DEVICES=0 python simplest_example.py' - } - } - stage ('Chatbot test') { - steps { - sh 'cd examples/start_here && CUDA_VISIBLE_DEVICES=1 python chatbot_example.py' - } + stage('L1: Parallel NLP Examples 1') { + when { + anyOf{ + branch 'master' + changeRequest() } } - } - - - - stage('Parallel NLP Examples 1') { failFast true parallel { stage ('Text Classification with BERT Test') { @@ -95,7 +111,13 @@ pipeline { } - stage('Parallel NLP Examples 2') { + stage('L1: Parallel NLP Examples 2') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true parallel { stage('Token Classification Training/Inference Test') { @@ -115,9 +137,15 @@ pipeline { } } - stage('Parallel NLP-Squad') { + stage('L1: Parallel NLP-Squad') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true - parallel { + parallel { stage('BERT Squad v1.1') { steps { sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=0 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case' @@ -135,8 +163,13 @@ pipeline { } } - - stage('Parallel NLP-Examples 3') { + stage('L1: Parallel NLP-Examples 3') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true parallel { stage('asr_processing') { @@ -156,7 +189,13 @@ pipeline { } } - stage('NLP-Intent Detection/SLot Tagging Examples - Multi-GPU') { + stage('L1: NLP-Intent Detection/SLot Tagging Examples - Multi-GPU') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true steps { sh 'cd examples/nlp/intent_detection_slot_tagging && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 joint_intent_slot_with_bert.py --num_gpus=2 --num_epochs=1 --max_seq_length=50 --dataset_name=jarvis-retail --data_dir=/home/mrjenkins/TestData/nlp/retail/ --eval_file_prefix=eval --batch_size=10 --num_train_samples=-1 --do_lower_case --shuffle_data --work_dir=outputs' @@ -166,15 +205,27 @@ pipeline { } } - stage('NLP-NMT Example') { + stage('L1: NLP-NMT Example') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true steps { - sh 'cd examples/nlp/neural_machine_translation/ && CUDA_VISIBLE_DEVICES=0 python machine_translation_tutorial.py --max_steps 100' + sh 'cd examples/nlp/neural_machine_translation/ && CUDA_VISIBLE_DEVICES=0 python machine_translation_tutorial.py --max_steps 100' sh 'rm -rf examples/nlp/neural_machine_translation/outputs' } } - stage('Parallel Stage Jasper / GAN') { + stage('L1: Parallel Stage Jasper / GAN') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true parallel { // stage('Jasper AN4 O1') { @@ -206,7 +257,13 @@ pipeline { // } // } - stage('Multi-GPU test') { + stage('L1: Multi-GPU Jasper test') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true parallel { stage('Jasper AN4 2 GPUs') { @@ -218,7 +275,13 @@ pipeline { } - stage('TTS Tests') { + stage('L1: TTS Tests') { + when { + anyOf{ + branch 'master' + changeRequest() + } + } failFast true steps { sh 'cd examples/tts && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 tacotron2.py --max_steps=51 --model_config=configs/tacotron2.yaml --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --amp_opt_level=O1 --eval_freq=50' diff --git a/examples/asr/jasper.py b/examples/asr/jasper.py index bb00ffa304ef..2e276ed64f23 100644 --- a/examples/asr/jasper.py +++ b/examples/asr/jasper.py @@ -44,9 +44,7 @@ def parse_args(): parser.add_argument("--beta1", default=0.95, type=float) parser.add_argument("--beta2", default=0.25, type=float) parser.add_argument("--warmup_steps", default=0, type=int) - parser.add_argument( - "--load_dir", default=None, type=str, help="directory with pre-trained checkpoint", - ) + parser.add_argument("--load_dir", default=None, type=str, help="directory with pre-trained checkpoint") args = parser.parse_args() From 72f58bbf23cea700f75b0c91d66539f6055a9293 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 13 Feb 2020 10:27:22 -0800 Subject: [PATCH 67/70] unjenkinsing Signed-off-by: Jason --- Jenkinsfile | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bfa626d141ad..c37479f37ed8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,14 +65,14 @@ pipeline { parallel { stage('BERT on the fly preprocessing') { steps { - sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=0 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wikitext-2 --dataset_name wikitext-2 --work_dir outputs/bert_lm/wikitext2 --batch_size 64 --lr 0.01 --lr_policy CosineAnnealing --lr_warmup_proportion 0.05 --tokenizer sentence-piece --vocab_size 3200 --hidden_size 768 --intermediate_size 3072 --num_hidden_layers 6 --num_attention_heads 12 --hidden_act "gelu" --save_step_freq 200 --sample_size 10000000 --mask_probability 0.15 --short_seq_prob 0.1 --max_steps=300' + sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=0 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/TestData/nlp/wikitext-2 --dataset_name wikitext-2 --work_dir outputs/bert_lm/wikitext2 --batch_size 64 --lr 0.01 --lr_policy CosineAnnealing --lr_warmup_proportion 0.05 --tokenizer sentence-piece --vocab_size 3200 --hidden_size 768 --intermediate_size 3072 --num_hidden_layers 6 --num_attention_heads 12 --hidden_act "gelu" --save_step_freq 200 --sample_size 10000000 --mask_probability 0.15 --short_seq_prob 0.1 --max_steps=300' sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wikitext2/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 8.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/language_modeling/outputs/wikitext2' } } stage('BERT offline preprocessing') { steps { - sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=1 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/mrjenkins/TestData/nlp/wiki_book_mini --work_dir outputs/bert_lm/wiki_book --batch_size 8 --config_file /home/mrjenkins/TestData/nlp/bert_configs/uncased_L-12_H-768_A-12.json --save_step_freq 200 --max_steps 300 --num_gpus 1 --batches_per_step 1 --lr_policy SquareRootAnnealing --beta2 0.999 --beta1 0.9 --lr_warmup_proportion 0.01 --optimizer adam_w --weight_decay 0.01 --lr 0.875e-4 --preprocessed_data ' + sh 'cd examples/nlp/language_modeling && CUDA_VISIBLE_DEVICES=1 python bert_pretraining.py --amp_opt_level O1 --data_dir /home/TestData/nlp/wiki_book_mini --work_dir outputs/bert_lm/wiki_book --batch_size 8 --config_file /home/TestData/nlp/bert_configs/uncased_L-12_H-768_A-12.json --save_step_freq 200 --max_steps 300 --num_gpus 1 --batches_per_step 1 --lr_policy SquareRootAnnealing --beta2 0.999 --beta1 0.9 --lr_warmup_proportion 0.01 --optimizer adam_w --weight_decay 0.01 --lr 0.875e-4 --preprocessed_data ' sh 'cd examples/nlp/language_modeling && LOSS=$(cat outputs/bert_lm/wiki_book/log_globalrank-0_localrank-0.txt | grep "Loss" |tail -n 1| awk \'{print \$7}\' | egrep -o "[0-9.]+" ) && echo $LOSS && if [ $(echo "$LOSS < 15.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/language_modeling/outputs/wiki_book' } @@ -91,19 +91,19 @@ pipeline { parallel { stage ('Text Classification with BERT Test') { steps { - sh 'cd examples/nlp/text_classification && CUDA_VISIBLE_DEVICES=0 python text_classification_with_bert.py --num_epochs=1 --max_seq_length=50 --dataset_name=jarvis --data_dir=/home/mrjenkins/TestData/nlp/retail/ --eval_file_prefix=eval --batch_size=10 --num_train_samples=-1 --do_lower_case --shuffle_data --work_dir=outputs' + sh 'cd examples/nlp/text_classification && CUDA_VISIBLE_DEVICES=0 python text_classification_with_bert.py --num_epochs=1 --max_seq_length=50 --dataset_name=jarvis --data_dir=/home/TestData/nlp/retail/ --eval_file_prefix=eval --batch_size=10 --num_train_samples=-1 --do_lower_case --shuffle_data --work_dir=outputs' sh 'rm -rf examples/nlp/text_classification/outputs' } } stage ('Dialogue State Tracking - TRADE - Multi-GPUs') { steps { - sh 'cd examples/nlp/dialogue_state_tracking && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 dialogue_state_tracking_trade.py --batch_size=10 --eval_batch_size=10 --num_train_samples=-1 --num_eval_samples=-1 --num_epochs=1 --dropout=0.2 --eval_file_prefix=test --shuffle_data --num_gpus=2 --lr=0.001 --grad_norm_clip=10 --work_dir=outputs --data_dir=/home/mrjenkins/TestData/nlp/multiwoz2.1' + sh 'cd examples/nlp/dialogue_state_tracking && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 dialogue_state_tracking_trade.py --batch_size=10 --eval_batch_size=10 --num_train_samples=-1 --num_eval_samples=-1 --num_epochs=1 --dropout=0.2 --eval_file_prefix=test --shuffle_data --num_gpus=2 --lr=0.001 --grad_norm_clip=10 --work_dir=outputs --data_dir=/home/TestData/nlp/multiwoz2.1' sh 'rm -rf examples/nlp/dialogue_state_tracking/outputs' } } stage ('GLUE Benchmark Test') { steps { - sh 'cd examples/nlp/glue_benchmark && CUDA_VISIBLE_DEVICES=1 python glue_benchmark_with_bert.py --data_dir /home/mrjenkins/TestData/nlp/glue_fake/MRPC --work_dir glue_output --save_step_freq -1 --num_epochs 1 --task_name mrpc --batch_size 2' + sh 'cd examples/nlp/glue_benchmark && CUDA_VISIBLE_DEVICES=1 python glue_benchmark_with_bert.py --data_dir /home/TestData/nlp/glue_fake/MRPC --work_dir glue_output --save_step_freq -1 --num_epochs 1 --task_name mrpc --batch_size 2' sh 'rm -rf examples/nlp/glue_benchmark/glue_output' } } @@ -122,15 +122,15 @@ pipeline { parallel { stage('Token Classification Training/Inference Test') { steps { - sh 'cd examples/nlp/token_classification && CUDA_VISIBLE_DEVICES=0 python token_classification.py --data_dir /home/mrjenkins/TestData/nlp/token_classification_punctuation/ --batch_size 2 --num_epochs 1 --save_epoch_freq 1 --work_dir token_classification_output --pretrained_bert_model bert-base-cased' - sh 'cd examples/nlp/token_classification && DATE_F=$(ls token_classification_output/) && CUDA_VISIBLE_DEVICES=0 python token_classification_infer.py --work_dir token_classification_output/$DATE_F/checkpoints/ --labels_dict /home/mrjenkins/TestData/nlp/token_classification_punctuation/label_ids.csv --pretrained_bert_model bert-base-cased' + sh 'cd examples/nlp/token_classification && CUDA_VISIBLE_DEVICES=0 python token_classification.py --data_dir /home/TestData/nlp/token_classification_punctuation/ --batch_size 2 --num_epochs 1 --save_epoch_freq 1 --work_dir token_classification_output --pretrained_bert_model bert-base-cased' + sh 'cd examples/nlp/token_classification && DATE_F=$(ls token_classification_output/) && CUDA_VISIBLE_DEVICES=0 python token_classification_infer.py --work_dir token_classification_output/$DATE_F/checkpoints/ --labels_dict /home/TestData/nlp/token_classification_punctuation/label_ids.csv --pretrained_bert_model bert-base-cased' sh 'rm -rf examples/nlp/token_classification/token_classification_output' } } stage ('Punctuation and Classification Training/Inference Test') { steps { - sh 'cd examples/nlp/token_classification && CUDA_VISIBLE_DEVICES=1 python punctuation_capitalization.py --data_dir /home/mrjenkins/TestData/nlp/token_classification_punctuation/ --work_dir punctuation_output --save_epoch_freq 1 --num_epochs 1 --save_step_freq -1 --batch_size 2' - sh 'cd examples/nlp/token_classification && DATE_F=$(ls punctuation_output/) && DATA_DIR="/home/mrjenkins/TestData/nlp/token_classification_punctuation" && CUDA_VISIBLE_DEVICES=1 python punctuation_capitalization_infer.py --checkpoints_dir punctuation_output/$DATE_F/checkpoints/ --punct_labels_dict $DATA_DIR/punct_label_ids.csv --capit_labels_dict $DATA_DIR/capit_label_ids.csv' + sh 'cd examples/nlp/token_classification && CUDA_VISIBLE_DEVICES=1 python punctuation_capitalization.py --data_dir /home/TestData/nlp/token_classification_punctuation/ --work_dir punctuation_output --save_epoch_freq 1 --num_epochs 1 --save_step_freq -1 --batch_size 2' + sh 'cd examples/nlp/token_classification && DATE_F=$(ls punctuation_output/) && DATA_DIR="/home/TestData/nlp/token_classification_punctuation" && CUDA_VISIBLE_DEVICES=1 python punctuation_capitalization_infer.py --checkpoints_dir punctuation_output/$DATE_F/checkpoints/ --punct_labels_dict $DATA_DIR/punct_label_ids.csv --capit_labels_dict $DATA_DIR/capit_label_ids.csv' sh 'rm -rf examples/nlp/token_classification/punctuation_output' } } @@ -148,16 +148,16 @@ pipeline { parallel { stage('BERT Squad v1.1') { steps { - sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=0 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case' + sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=0 python question_answering_squad.py --amp_opt_level O1 --train_file /home/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case' sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv1/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' - sh 'rm -rf examples/nlp/question_answering/outputs/squadv1 && rm -rf /home/mrjenkins/TestData/nlp/squad_mini/v1.1/*cache*' + sh 'rm -rf examples/nlp/question_answering/outputs/squadv1 && rm -rf /home/TestData/nlp/squad_mini/v1.1/*cache*' } } stage('BERT Squad v2.0') { steps { - sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v2.0/train-v2.0.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v2.0/dev-v2.0.json --work_dir outputs/squadv2 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --version_2_with_negative' + sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/TestData/nlp/squad_mini/v2.0/train-v2.0.json --dev_file /home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json --work_dir outputs/squadv2 --batch_size 8 --save_step_freq 300 --num_epochs 3 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --version_2_with_negative' sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv2/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' - sh 'rm -rf examples/nlp/question_answering/outputs/squadv2 && rm -rf /home/mrjenkins/TestData/nlp/squad_mini/v2.0/*cache*' + sh 'rm -rf examples/nlp/question_answering/outputs/squadv2 && rm -rf /home/TestData/nlp/squad_mini/v2.0/*cache*' } } } @@ -174,16 +174,16 @@ pipeline { parallel { stage('asr_processing') { steps { - sh 'cd examples/nlp/asr_postprocessor && CUDA_VISIBLE_DEVICES=0 python asr_postprocessor.py --data_dir=/home/mrjenkins/TestData/nlp/asr_postprocessor/pred_real --restore_from=/home/mrjenkins/TestData/nlp/asr_postprocessor/bert-base-uncased_decoder.pt --max_steps=25 --batch_size=64' + sh 'cd examples/nlp/asr_postprocessor && CUDA_VISIBLE_DEVICES=0 python asr_postprocessor.py --data_dir=/home/TestData/nlp/asr_postprocessor/pred_real --restore_from=/home/TestData/nlp/asr_postprocessor/bert-base-uncased_decoder.pt --max_steps=25 --batch_size=64' sh 'cd examples/nlp/asr_postprocessor && WER=$(cat outputs/asr_postprocessor/log_globalrank-0_localrank-0.txt | grep "Validation WER" | tail -n 1 | egrep -o "[0-9.]+" | tail -n 1) && echo $WER && if [ $(echo "$WER < 25.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' sh 'rm -rf examples/nlp/asr_postprocessor/outputs' } } stage('Roberta Squad v1.1') { steps { - sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/mrjenkins/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1_roberta --batch_size 2 --save_step_freq 500 --num_epochs 1 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --model_type roberta --pretrained_model_name roberta-base' + sh 'cd examples/nlp/question_answering && CUDA_VISIBLE_DEVICES=1 python question_answering_squad.py --amp_opt_level O1 --train_file /home/TestData/nlp/squad_mini/v1.1/train-v1.1.json --dev_file /home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json --work_dir outputs/squadv1_roberta --batch_size 2 --save_step_freq 500 --num_epochs 1 --lr_policy WarmupAnnealing --lr 3e-5 --do_lower_case --model_type roberta --pretrained_model_name roberta-base' sh 'cd examples/nlp/question_answering && FSCORE=$(cat outputs/squadv1_roberta/log_globalrank-0_localrank-0.txt | grep "f1" |tail -n 1 |egrep -o "[0-9.]+"|tail -n 1 ) && echo $FSCORE && if [ $(echo "$FSCORE > 50.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' - sh 'rm -rf examples/nlp/question_answering/outputs/squadv1_roberta && rm -rf /home/mrjenkins/TestData/nlp/squad_mini/v1.1/*cache*' + sh 'rm -rf examples/nlp/question_answering/outputs/squadv1_roberta && rm -rf /home/TestData/nlp/squad_mini/v1.1/*cache*' } } } @@ -198,9 +198,9 @@ pipeline { } failFast true steps { - sh 'cd examples/nlp/intent_detection_slot_tagging && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 joint_intent_slot_with_bert.py --num_gpus=2 --num_epochs=1 --max_seq_length=50 --dataset_name=jarvis-retail --data_dir=/home/mrjenkins/TestData/nlp/retail/ --eval_file_prefix=eval --batch_size=10 --num_train_samples=-1 --do_lower_case --shuffle_data --work_dir=outputs' - sh 'cd examples/nlp/intent_detection_slot_tagging && TASK_NAME=$(ls outputs/) && DATE_F=$(ls outputs/$TASK_NAME/) && CHECKPOINT_DIR=outputs/$TASK_NAME/$DATE_F/checkpoints/ && CUDA_VISIBLE_DEVICES=0 python joint_intent_slot_infer.py --work_dir $CHECKPOINT_DIR --eval_file_prefix=eval --dataset_name=jarvis-retail --data_dir=/home/mrjenkins/TestData/nlp/retail/ --batch_size=10' - sh 'cd examples/nlp/intent_detection_slot_tagging && TASK_NAME=$(ls outputs/) && DATE_F=$(ls outputs/$TASK_NAME/) && CHECKPOINT_DIR=outputs/$TASK_NAME/$DATE_F/checkpoints/ && CUDA_VISIBLE_DEVICES=0 python joint_intent_slot_infer_b1.py --data_dir=/home/mrjenkins/TestData/nlp/retail/ --work_dir $CHECKPOINT_DIR --dataset_name=jarvis-retail --query="how much is it?"' + sh 'cd examples/nlp/intent_detection_slot_tagging && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 joint_intent_slot_with_bert.py --num_gpus=2 --num_epochs=1 --max_seq_length=50 --dataset_name=jarvis-retail --data_dir=/home/TestData/nlp/retail/ --eval_file_prefix=eval --batch_size=10 --num_train_samples=-1 --do_lower_case --shuffle_data --work_dir=outputs' + sh 'cd examples/nlp/intent_detection_slot_tagging && TASK_NAME=$(ls outputs/) && DATE_F=$(ls outputs/$TASK_NAME/) && CHECKPOINT_DIR=outputs/$TASK_NAME/$DATE_F/checkpoints/ && CUDA_VISIBLE_DEVICES=0 python joint_intent_slot_infer.py --work_dir $CHECKPOINT_DIR --eval_file_prefix=eval --dataset_name=jarvis-retail --data_dir=/home/TestData/nlp/retail/ --batch_size=10' + sh 'cd examples/nlp/intent_detection_slot_tagging && TASK_NAME=$(ls outputs/) && DATE_F=$(ls outputs/$TASK_NAME/) && CHECKPOINT_DIR=outputs/$TASK_NAME/$DATE_F/checkpoints/ && CUDA_VISIBLE_DEVICES=0 python joint_intent_slot_infer_b1.py --data_dir=/home/TestData/nlp/retail/ --work_dir $CHECKPOINT_DIR --dataset_name=jarvis-retail --query="how much is it?"' sh 'rm -rf examples/nlp/intent_detection_slot_tagging/outputs' } } @@ -235,12 +235,12 @@ pipeline { // } stage('GAN O2') { steps { - sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O2 --num_epochs=3 --train_dataset=/home/mrjenkins/TestData/' + sh 'cd examples/image && CUDA_VISIBLE_DEVICES=0 python gan.py --amp_opt_level=O2 --num_epochs=3 --train_dataset=/home/TestData/' } } stage('Jasper AN4 O2') { steps { - sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=1 python jasper_an4.py --amp_opt_level=O2 --num_epochs=35 --test_after_training --work_dir=O2 --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --eval_datasets=/home/mrjenkins/TestData/an4_dataset/an4_val.json' + sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=1 python jasper_an4.py --amp_opt_level=O2 --num_epochs=35 --test_after_training --work_dir=O2 --train_dataset=/home/TestData/an4_dataset/an4_train.json --eval_datasets=/home/TestData/an4_dataset/an4_val.json' } } } @@ -268,7 +268,7 @@ pipeline { parallel { stage('Jasper AN4 2 GPUs') { steps { - sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 jasper_an4.py --num_epochs=40 --batch_size=24 --work_dir=multi_gpu --test_after_training --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --eval_datasets=/home/mrjenkins/TestData/an4_dataset/an4_val.json' + sh 'cd examples/asr && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 jasper_an4.py --num_epochs=40 --batch_size=24 --work_dir=multi_gpu --test_after_training --train_dataset=/home/TestData/an4_dataset/an4_train.json --eval_datasets=/home/TestData/an4_dataset/an4_val.json' } } } @@ -284,10 +284,10 @@ pipeline { } failFast true steps { - sh 'cd examples/tts && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 tacotron2.py --max_steps=51 --model_config=configs/tacotron2.yaml --train_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --amp_opt_level=O1 --eval_freq=50' + sh 'cd examples/tts && CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 tacotron2.py --max_steps=51 --model_config=configs/tacotron2.yaml --train_dataset=/home/TestData/an4_dataset/an4_train.json --amp_opt_level=O1 --eval_freq=50' sh 'cd examples/tts && TTS_CHECKPOINT_DIR=$(ls | grep "Tacotron2") && echo $TTS_CHECKPOINT_DIR && LOSS=$(cat $TTS_CHECKPOINT_DIR/log_globalrank-0_localrank-0.txt | grep -o -E "Loss[ :0-9.]+" | grep -o -E "[0-9.]+" | tail -n 1) && echo $LOSS && if [ $(echo "$LOSS < 3.0" | bc -l) -eq 1 ]; then echo "SUCCESS" && exit 0; else echo "FAILURE" && exit 1; fi' // sh 'cd examples/tts && TTS_CHECKPOINT_DIR=$(ls | grep "Tacotron2") && cp ../asr/multi_gpu/checkpoints/* $TTS_CHECKPOINT_DIR/checkpoints' - // sh 'CUDA_VISIBLE_DEVICES=0 python tacotron2_an4_test.py --model_config=configs/tacotron2.yaml --eval_dataset=/home/mrjenkins/TestData/an4_dataset/an4_train.json --jasper_model_config=../asr/configs/jasper_an4.yaml --load_dir=$TTS_CHECKPOINT_DIR/checkpoints' + // sh 'CUDA_VISIBLE_DEVICES=0 python tacotron2_an4_test.py --model_config=configs/tacotron2.yaml --eval_dataset=/home/TestData/an4_dataset/an4_train.json --jasper_model_config=../asr/configs/jasper_an4.yaml --load_dir=$TTS_CHECKPOINT_DIR/checkpoints' } } From 403238f82d26879ba5fca53fbf75b3cdc70fb49b Mon Sep 17 00:00:00 2001 From: Adriana Flores Date: Thu, 13 Feb 2020 12:31:53 -0700 Subject: [PATCH 68/70] Issue 357 (#360) * Issue 357: Fix to export jasper to onnx - logger and factory creation Signed-off-by: adriana * Issue 357: Changes to jasper_eval.py - added amp_opt_level, cache=False and formatting Signed-off-by: adriana * Issue 357: Updated changes for PR Signed-off-by: adriana --- examples/asr/jasper_eval.py | 53 ++++++++++++++++---------------- scripts/export_jasper_to_onnx.py | 6 +++- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/examples/asr/jasper_eval.py b/examples/asr/jasper_eval.py index b4b16699d13f..9c5fac4eb36d 100644 --- a/examples/asr/jasper_eval.py +++ b/examples/asr/jasper_eval.py @@ -16,16 +16,20 @@ def main(): parser = argparse.ArgumentParser(description='Jasper') - parser.add_argument("--local_rank", default=None, type=int) - parser.add_argument("--batch_size", default=64, type=int) + # model params parser.add_argument("--model_config", type=str, required=True) parser.add_argument("--eval_datasets", type=str, required=True) parser.add_argument("--load_dir", type=str, required=True) + # run params + parser.add_argument("--local_rank", default=None, type=int) + parser.add_argument("--batch_size", default=64, type=int) + parser.add_argument("--amp_opt_level", default="O1", type=str) + # store results parser.add_argument("--save_logprob", default=None, type=str) + + # lm inference parameters parser.add_argument("--lm_path", default=None, type=str) - parser.add_argument( - '--alpha', default=2.0, type=float, help='value of LM weight', required=False, - ) + parser.add_argument('--alpha', default=2.0, type=float, help='value of LM weight', required=False) parser.add_argument( '--alpha_max', type=float, @@ -33,11 +37,9 @@ def main(): required=False, ) parser.add_argument( - '--alpha_step', type=float, help='step for LM weight\'s tuning in \'eval\' mode', required=False, default=0.1, - ) - parser.add_argument( - '--beta', default=1.5, type=float, help='value of word count weight', required=False, + '--alpha_step', type=float, help='step for LM weight\'s tuning in \'eval\' mode', required=False, default=0.1 ) + parser.add_argument('--beta', default=1.5, type=float, help='value of word count weight', required=False) parser.add_argument( '--beta_max', type=float, @@ -71,7 +73,7 @@ def main(): neural_factory = nemo.core.NeuralModuleFactory( backend=nemo.core.Backend.PyTorch, local_rank=args.local_rank, - optimization_level=nemo.core.Optimization.mxprO1, + optimization_level=args.amp_opt_level, placement=device, ) @@ -102,13 +104,13 @@ def main(): nemo.logging.info('Evaluating {0} examples'.format(N)) data_preprocessor = nemo_asr.AudioToMelSpectrogramPreprocessor( - sample_rate=sample_rate, **jasper_params["AudioToMelSpectrogramPreprocessor"], + sample_rate=sample_rate, **jasper_params["AudioToMelSpectrogramPreprocessor"] ) jasper_encoder = nemo_asr.JasperEncoder( - feat_in=jasper_params["AudioToMelSpectrogramPreprocessor"]["features"], **jasper_params["JasperEncoder"], + feat_in=jasper_params["AudioToMelSpectrogramPreprocessor"]["features"], **jasper_params["JasperEncoder"] ) jasper_decoder = nemo_asr.JasperDecoderForCTC( - feat_in=jasper_params["JasperEncoder"]["jasper"][-1]["filters"], num_classes=len(vocab), + feat_in=jasper_params["JasperEncoder"]["jasper"][-1]["filters"], num_classes=len(vocab) ) greedy_decoder = nemo_asr.GreedyCTCDecoder() @@ -120,27 +122,25 @@ def main(): ) nemo.logging.info('================================') - (audio_signal_e1, a_sig_length_e1, transcript_e1, transcript_len_e1,) = data_layer() + # Define inference DAG + audio_signal_e1, a_sig_length_e1, transcript_e1, transcript_len_e1 = data_layer() processed_signal_e1, p_length_e1 = data_preprocessor(input_signal=audio_signal_e1, length=a_sig_length_e1) encoded_e1, encoded_len_e1 = jasper_encoder(audio_signal=processed_signal_e1, length=p_length_e1) log_probs_e1 = jasper_decoder(encoder_output=encoded_e1) predictions_e1 = greedy_decoder(log_probs=log_probs_e1) - eval_tensors = [ - log_probs_e1, - predictions_e1, - transcript_e1, - transcript_len_e1, - encoded_len_e1, - ] + eval_tensors = [log_probs_e1, predictions_e1, transcript_e1, transcript_len_e1, encoded_len_e1] - evaluated_tensors = neural_factory.infer(tensors=eval_tensors, checkpoint_dir=load_dir, cache=True) + # inference + evaluated_tensors = neural_factory.infer(tensors=eval_tensors, checkpoint_dir=load_dir, cache=False) greedy_hypotheses = post_process_predictions(evaluated_tensors[1], vocab) references = post_process_transcripts(evaluated_tensors[2], evaluated_tensors[3], vocab) + wer = word_error_rate(hypotheses=greedy_hypotheses, references=references) nemo.logging.info("Greedy WER {:.2f}%".format(wer * 100)) + # language model if args.lm_path: if args.alpha_max is None: args.alpha_max = args.alpha @@ -168,7 +168,7 @@ def main(): ) beam_predictions_e1 = beam_search_with_lm(log_probs=log_probs_e1, log_probs_length=encoded_len_e1) - evaluated_tensors = neural_factory.infer(tensors=[beam_predictions_e1], use_cache=True, verbose=False,) + evaluated_tensors = neural_factory.infer(tensors=[beam_predictions_e1], use_cache=False, verbose=False) beam_hypotheses = [] # Over mini-batch @@ -176,10 +176,9 @@ def main(): # Over samples for j in i: beam_hypotheses.append(j[0][1]) - - wer = word_error_rate(hypotheses=beam_hypotheses, references=references) - nemo.logging.info("Beam WER {:.2f}%".format(wer * 100)) - beam_wers.append(((alpha, beta), wer * 100)) + lm_wer = word_error_rate(hypotheses=beam_hypotheses, references=references) + nemo.logging.info("Beam WER {:.2f}%".format(lm_wer * 100)) + beam_wers.append(((alpha, beta), lm_wer * 100)) nemo.logging.info('Beam WER for (alpha, beta)') nemo.logging.info('================================') diff --git a/scripts/export_jasper_to_onnx.py b/scripts/export_jasper_to_onnx.py index 84db7bddaf9a..dbb24023fa2f 100644 --- a/scripts/export_jasper_to_onnx.py +++ b/scripts/export_jasper_to_onnx.py @@ -7,6 +7,8 @@ import nemo import nemo.collections.asr as nemo_asr +logging = nemo.logging + def get_parser(): parser = argparse.ArgumentParser(description="Convert Jasper NeMo checkpoint to ONNX") @@ -58,10 +60,13 @@ def main( logging.info(" Num encoder input features: {}".format(num_encoder_input_features)) logging.info(" Num decoder input features: {}".format(num_decoder_input_features)) + nf = nemo.core.NeuralModuleFactory(create_tb_writer=False) + logging.info("Initializing models...") jasper_encoder = nemo_asr.JasperEncoder( feat_in=num_encoder_input_features, **jasper_model_definition['JasperEncoder'] ) + jasper_decoder = nemo_asr.JasperDecoderForCTC( feat_in=num_decoder_input_features, num_classes=len(jasper_model_definition['labels']), ) @@ -83,7 +88,6 @@ def main( jasper_encoder.restore_from(nn_encoder) jasper_decoder.restore_from(nn_decoder) - nf = nemo.core.NeuralModuleFactory(create_tb_writer=False) logging.info("Exporting encoder...") nf.deployment_export( jasper_encoder, From 40915ef86e5c89b7628d26f1a4dfde1a5a1ec5e1 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 13 Feb 2020 13:06:37 -0800 Subject: [PATCH 69/70] adding link to ngc bert checkpoint to documentation Signed-off-by: Yang Zhang --- docs/sources/source/nlp/bert_pretraining.rst | 6 ++++++ docs/sources/source/nlp/joint_intent_slot_filling.rst | 4 ++++ docs/sources/source/nlp/ner.rst | 6 ++++++ docs/sources/source/nlp/punctuation.rst | 2 ++ docs/sources/source/nlp/question_answering.rst | 6 ++++++ examples/nlp/language_modeling/bert_pretraining.py | 9 +++++++++ 6 files changed, 33 insertions(+) diff --git a/docs/sources/source/nlp/bert_pretraining.rst b/docs/sources/source/nlp/bert_pretraining.rst index 26946ea836e8..389f6a307466 100644 --- a/docs/sources/source/nlp/bert_pretraining.rst +++ b/docs/sources/source/nlp/bert_pretraining.rst @@ -6,6 +6,12 @@ Make sure you have ``nemo`` and ``nemo_nlp`` installed before starting this tuto The code used in this tutorial can be found at ``examples/nlp/language_modeling/bert_pretraining.py``. +.. tip:: + Pretrained BERT models can be found at + `https://ngc.nvidia.com/catalog/models/nvidia:bertlargeuncasedfornemo `__ + `https://ngc.nvidia.com/catalog/models/nvidia:bertbaseuncasedfornemo `__ + `https://ngc.nvidia.com/catalog/models/nvidia:bertbasecasedfornemo `__ + Introduction ------------ diff --git a/docs/sources/source/nlp/joint_intent_slot_filling.rst b/docs/sources/source/nlp/joint_intent_slot_filling.rst index 830a110eff41..57b82629b0be 100644 --- a/docs/sources/source/nlp/joint_intent_slot_filling.rst +++ b/docs/sources/source/nlp/joint_intent_slot_filling.rst @@ -9,6 +9,10 @@ There are four pre-trained BERT models that we can select from using the argumen using the script for loading pre-trained models from `pytorch_transformers`. See the list of available pre-trained models `here `__. +.. tip:: + + For pretraining BERT in NeMo and pretrained model checkpoints go to `BERT pretraining `__. + Preliminaries ------------- diff --git a/docs/sources/source/nlp/ner.rst b/docs/sources/source/nlp/ner.rst index c139a09fa912..8a44f3c48cb2 100644 --- a/docs/sources/source/nlp/ner.rst +++ b/docs/sources/source/nlp/ner.rst @@ -4,6 +4,12 @@ Tutorial Make sure you have ``nemo`` and ``nemo_nlp`` installed before starting this tutorial. See the :ref:`installation` section for more details. +.. tip:: + + For pretraining BERT in NeMo and pretrained model checkpoints go to `BERT pretraining `__. + + + Introduction ------------ diff --git a/docs/sources/source/nlp/punctuation.rst b/docs/sources/source/nlp/punctuation.rst index 5433f7847f48..6834eced59a2 100644 --- a/docs/sources/source/nlp/punctuation.rst +++ b/docs/sources/source/nlp/punctuation.rst @@ -7,6 +7,8 @@ An ASR system typically generates text with no punctuation and capitalization of .. tip:: We recommend you to try this example in Jupyter notebook examples/nlp/token_classification/PunctuationWithBERT.ipynb. + For pretraining BERT in NeMo and pretrained model checkpoints go to `BERT pretraining `__. + Task Description ---------------- diff --git a/docs/sources/source/nlp/question_answering.rst b/docs/sources/source/nlp/question_answering.rst index 80482c875c97..98ad59b76d7d 100644 --- a/docs/sources/source/nlp/question_answering.rst +++ b/docs/sources/source/nlp/question_answering.rst @@ -13,6 +13,12 @@ The pretrained back-bone models can be specified by `--model_type` and the speci See the list of available pre-trained models `here `__. +.. tip:: + + For pretraining BERT in NeMo and pretrained model checkpoints go to `BERT pretraining `__. + + + Preliminaries ------------- diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index eaf40dc454ed..7ca3871b7d32 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -70,6 +70,15 @@ 350000 iterations on a DGX1 with 8 V100 32GB GPUs with AMP O1 optimization should finish under 5 days and yield an MRPC score of ACC/F1 85.05/89.35. + +More information about BERT pretraining can be found at +https://nvidia.github.io/NeMo/nlp/bert_pretraining.html + +Pretrained BERT models can be found at +https://ngc.nvidia.com/catalog/models/nvidia:bertlargeuncasedfornemo +https://ngc.nvidia.com/catalog/models/nvidia:bertbaseuncasedfornemo +https://ngc.nvidia.com/catalog/models/nvidia:bertbasecasedfornemo + """ import argparse import math From d48f605dbf2949a4d85eecdb8a1c0d9c16b9391f Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 13 Feb 2020 13:19:58 -0800 Subject: [PATCH 70/70] deleted squad unit test (#367) Signed-off-by: Yang Zhang --- tests/nlp/test_squad.py | 44 ----------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 tests/nlp/test_squad.py diff --git a/tests/nlp/test_squad.py b/tests/nlp/test_squad.py deleted file mode 100644 index aa69076185c1..000000000000 --- a/tests/nlp/test_squad.py +++ /dev/null @@ -1,44 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright 2020 NVIDIA. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -import json -import os -import shutil - -from examples.nlp.scripts.get_squad import SquadDownloader - -import nemo -import nemo.collections.nlp as nemo_nlp -from tests.common_setup import NeMoUnitTest - -logging = nemo.logging - - -class TestSquad(NeMoUnitTest): - def test_setup_squad(self): - pretrained_bert_model = 'bert-base-uncased' - tokenizer = nemo_nlp.data.NemoBertTokenizer(pretrained_bert_model) - neural_factory = nemo.core.NeuralModuleFactory( - backend=nemo.core.Backend.PyTorch, local_rank=None, create_tb_writer=False - ) - model = nemo_nlp.nm.trainables.huggingface.BERT(pretrained_model_name=pretrained_bert_model) - hidden_size = model.hidden_size - qa_head = nemo_nlp.nm.trainables.token_classification_nm.TokenClassifier( - hidden_size=hidden_size, num_classes=2, num_layers=1, log_softmax=False - ) - squad_loss = nemo_nlp.nm.losses.QuestionAnsweringLoss()