From caddc35cee986ffdc396326ebba722728d4bd833 Mon Sep 17 00:00:00 2001 From: Stanislav Beliaev Date: Wed, 22 Jan 2020 16:45:48 -0800 Subject: [PATCH] Pure isort formatting. Signed-off-by: Stanislav Beliaev --- collections/nemo_asr/nemo_asr/__init__.py | 2 +- .../nemo_asr/nemo_asr/audio_preprocessing.py | 15 ++++++----- .../nemo_asr/nemo_asr/beam_search_decoder.py | 4 +-- collections/nemo_asr/nemo_asr/data_layer.py | 2 +- .../nemo_asr/nemo_asr/greedy_ctc_decoder.py | 4 +-- collections/nemo_asr/nemo_asr/helpers.py | 1 + collections/nemo_asr/nemo_asr/jasper.py | 15 ++++++----- collections/nemo_asr/nemo_asr/las/helpers.py | 2 +- collections/nemo_asr/nemo_asr/las/misc.py | 4 +-- collections/nemo_asr/nemo_asr/losses.py | 4 +-- .../nemo_asr/nemo_asr/parts/cleaners.py | 2 +- .../nemo_asr/nemo_asr/parts/collections.py | 5 ++-- .../nemo_asr/nemo_asr/parts/dataset.py | 3 +-- .../nemo_asr/nemo_asr/parts/features.py | 6 +++-- collections/nemo_asr/nemo_asr/parts/jasper.py | 2 +- .../nemo_asr/nemo_asr/parts/manifest.py | 2 +- .../nemo_asr/nemo_asr/parts/parsers.py | 3 ++- .../nemo_asr/nemo_asr/parts/perturb.py | 3 +-- collections/nemo_nlp/nemo_nlp/__init__.py | 6 ++--- .../nemo_nlp/nemo_nlp/data/data_layers.py | 3 ++- .../data/datasets/bert_pretraining.py | 1 + .../nemo_nlp/nemo_nlp/data/datasets/glue.py | 3 ++- .../datasets/punctuation_capitalization.py | 3 ++- .../data/datasets/sentence_classification.py | 3 ++- .../nemo_nlp/nemo_nlp/data/datasets/squad.py | 11 ++++---- .../data/datasets/token_classification.py | 3 ++- .../nemo_nlp/data/datasets/translation.py | 2 +- .../nemo_nlp/nemo_nlp/data/datasets/utils.py | 8 +++--- .../nemo_nlp/data/tokenizers/__init__.py | 6 ++--- .../data/tokenizers/bert_tokenizer.py | 6 +++-- .../data/tokenizers/gpt2_tokenizer.py | 3 ++- .../nemo_nlp/data/tokenizers/spc_tokenizer.py | 1 + .../data/tokenizers/tokenizer_spec.py | 2 +- .../data/tokenizers/yttm_tokenizer.py | 1 + collections/nemo_nlp/nemo_nlp/data/utils.py | 3 ++- .../nemo_nlp/nemo_nlp/huggingface/bert.py | 6 ++--- .../nemo_nlp/nemo_nlp/modules/classifiers.py | 3 +-- .../nemo_nlp/nemo_nlp/modules/losses.py | 5 ++-- .../nemo_nlp/modules/transformer_nm.py | 8 +++--- .../nemo_nlp/transformer/generators.py | 2 +- .../nemo_nlp/nemo_nlp/transformer/modules.py | 10 +++---- .../nemo_nlp/nemo_nlp/utils/__init__.py | 4 +-- .../utils/callbacks/bert_pretraining.py | 3 ++- .../nemo_nlp/nemo_nlp/utils/callbacks/glue.py | 5 ++-- .../utils/callbacks/joint_intent_slot.py | 5 ++-- .../utils/callbacks/language_modeling.py | 3 ++- .../callbacks/punctuation_capitalization.py | 2 +- .../callbacks/sentence_classification.py | 4 +-- .../nemo_nlp/utils/callbacks/translation.py | 1 - .../utils/metrics/fairseq_tokenizer.py | 2 +- .../nemo_nlp/utils/metrics/sacrebleu.py | 7 +++-- .../nemo_nlp/nemo_nlp/utils/nlp_utils.py | 5 ++-- .../nemo_simple_gan/__init__.py | 4 +-- .../nemo_simple_gan/nemo_simple_gan/gan.py | 14 +++++----- collections/nemo_tts/nemo_tts/__init__.py | 26 +++++++++---------- collections/nemo_tts/nemo_tts/data_layers.py | 2 ++ .../nemo_tts/nemo_tts/parts/__init__.py | 14 +++++----- .../nemo_tts/nemo_tts/parts/datasets.py | 3 +-- .../nemo_tts/nemo_tts/parts/tacotron2.py | 2 +- .../nemo_tts/nemo_tts/parts/waveglow.py | 2 +- .../nemo_tts/nemo_tts/tacotron2_modules.py | 5 ++-- .../nemo_tts/nemo_tts/waveglow_modules.py | 3 ++- docs/docs_zh/sources/source/conf.py | 1 + docs/sources/source/conf.py | 7 ++--- .../applications/asr_service/app/__init__.py | 5 ++-- .../applications/asr_service/app/routes.py | 19 +++++++------- examples/asr/experimental/garnet.py | 4 +-- examples/asr/experimental/garnet_rnnlm.py | 4 +-- examples/asr/jasper.py | 4 +-- examples/asr/jasper_aishell.py | 4 +-- examples/asr/jasper_aishell_infer.py | 2 +- examples/asr/jasper_an4.py | 8 +++--- examples/asr/jasper_eval.py | 4 +-- examples/asr/quartznet.py | 4 +-- examples/image/gan.py | 7 +++-- examples/image/resnet50.py | 16 +++++++----- examples/image/transfer_learning.py | 16 +++++++----- examples/nlp/asr_postprocessor.py | 7 +++-- examples/nlp/bert_pretraining.py | 11 ++++---- examples/nlp/glue_with_BERT.py | 17 +++++++----- examples/nlp/joint_intent_slot_infer.py | 3 +-- examples/nlp/joint_intent_slot_infer_b1.py | 1 - examples/nlp/joint_intent_slot_with_bert.py | 6 ++--- examples/nlp/nmt_tutorial.py | 5 ++-- examples/nlp/punctuation_capitalization.py | 7 +++-- examples/nlp/scripts/process_wiki_zh.py | 6 ++--- .../nlp/sentence_classification_with_bert.py | 9 +++---- examples/nlp/squad.py | 2 +- examples/nlp/token_classification.py | 7 +++-- examples/nlp/transformer_lm.py | 6 ++--- examples/start_here/chatbot_example.py | 3 ++- examples/start_here/chatbot_example2.py | 3 ++- examples/tts/tacotron2.py | 6 ++--- examples/tts/waveglow.py | 4 +-- nemo/nemo/__init__.py | 4 +-- nemo/nemo/backends/pytorch/__init__.py | 4 +-- nemo/nemo/backends/pytorch/actions.py | 16 +++++++----- nemo/nemo/backends/pytorch/common/losses.py | 4 +-- nemo/nemo/backends/pytorch/common/other.py | 2 +- nemo/nemo/backends/pytorch/common/rnn.py | 4 +-- nemo/nemo/backends/pytorch/common/search.py | 4 +-- .../nemo/backends/pytorch/common/zero_data.py | 2 +- nemo/nemo/backends/pytorch/module_wrapper.py | 2 +- nemo/nemo/backends/pytorch/nm.py | 6 ++--- .../pytorch/torchvision/data/image_folder.py | 4 +-- .../pytorch/tutorials/chatbot/modules.py | 6 ++--- nemo/nemo/backends/pytorch/tutorials/toys.py | 4 +-- nemo/nemo/core/callbacks.py | 4 +-- nemo/nemo/core/neural_factory.py | 9 ++++--- nemo/nemo/core/neural_modules.py | 12 ++++----- nemo/nemo/core/neural_types.py | 2 +- nemo/nemo/utils/exp_logging.py | 4 +-- nemo/nemo/utils/lr_policies.py | 5 ++-- scripts/build_lm_text.py | 4 +-- scripts/export_bert_to_trt.py | 3 ++- scripts/export_jasper_onnx_to_trt.py | 1 + scripts/export_jasper_to_onnx.py | 5 ++-- scripts/get_aishell_data.py | 6 ++--- scripts/get_databaker_data.py | 9 +++---- scripts/get_decoder_params_from_bert.py | 3 ++- scripts/get_librispeech_data.py | 11 ++++---- scripts/get_ljspeech_data.py | 1 + scripts/get_tatoeba_data.py | 1 - scripts/get_timit_data.py | 13 +++++----- scripts/process_aishell2_data.py | 4 +-- scripts/process_an4_data.py | 3 ++- scripts/process_fisher_data.py | 2 +- scripts/process_hub5_data.py | 6 ++--- tests/common_setup.py | 5 ++-- tests/context.py | 5 ++-- tests/other/jasper.py | 13 +++++----- tests/other/jasper_zero_dl.py | 7 ++--- tests/test_actions_api.py | 4 +-- tests/test_asr.py | 12 ++++++--- tests/test_bert.py | 4 +-- tests/test_infer.py | 2 +- tests/test_neural_factory.py | 3 ++- tests/test_neural_modules.py | 4 ++- tests/test_neural_types.py | 5 ++-- tests/test_policies.py | 2 +- tests/test_pytorch_trainers.py | 3 ++- tests/test_spc_tokenizer.py | 5 ++-- tests/test_squad.py | 7 ++--- tests/test_tts.py | 2 +- tests/test_tutorials_pytorch.py | 4 ++- tests/test_weight_share.py | 7 ++--- tests/test_zeroDS.py | 6 +++-- 147 files changed, 407 insertions(+), 365 deletions(-) diff --git a/collections/nemo_asr/nemo_asr/__init__.py b/collections/nemo_asr/nemo_asr/__init__.py index 1b810480b0603..172184b2ffabb 100644 --- a/collections/nemo_asr/nemo_asr/__init__.py +++ b/collections/nemo_asr/nemo_asr/__init__.py @@ -22,7 +22,7 @@ TranscriptDataLayer, ) from .greedy_ctc_decoder import GreedyCTCDecoder -from .jasper import JasperEncoder, JasperDecoderForCTC +from .jasper import JasperDecoderForCTC, JasperEncoder from .las.misc import JasperRNNConnector from .losses import CTCLossNM diff --git a/collections/nemo_asr/nemo_asr/audio_preprocessing.py b/collections/nemo_asr/nemo_asr/audio_preprocessing.py index 491d9b452baff..1fa0f987558c3 100644 --- a/collections/nemo_asr/nemo_asr/audio_preprocessing.py +++ b/collections/nemo_asr/nemo_asr/audio_preprocessing.py @@ -24,12 +24,19 @@ 'SpectrogramAugmentation', ] -from abc import abstractmethod import math import warnings +from abc import abstractmethod import torch +from nemo.backends.pytorch import NonTrainableNM +from nemo.core import Optimization +from nemo.core.neural_types import * + +from .parts.features import FilterbankFeatures +from .parts.spectr_augment import SpecAugment, SpecCutout + try: import torchaudio @@ -45,12 +52,6 @@ "will not work." ) -from nemo.backends.pytorch import NonTrainableNM -from nemo.core import Optimization -from nemo.core.neural_types import * -from .parts.features import FilterbankFeatures -from .parts.spectr_augment import SpecAugment, SpecCutout - class AudioPreprocessor(NonTrainableNM): """ diff --git a/collections/nemo_asr/nemo_asr/beam_search_decoder.py b/collections/nemo_asr/nemo_asr/beam_search_decoder.py index 8e3baadbfce20..a442f4e067403 100644 --- a/collections/nemo_asr/nemo_asr/beam_search_decoder.py +++ b/collections/nemo_asr/nemo_asr/beam_search_decoder.py @@ -7,11 +7,11 @@ from nemo.backends.pytorch.nm import NonTrainableNM from nemo.core import DeviceType from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) diff --git a/collections/nemo_asr/nemo_asr/data_layer.py b/collections/nemo_asr/nemo_asr/data_layer.py index a6a917a58551c..4091673065ab0 100644 --- a/collections/nemo_asr/nemo_asr/data_layer.py +++ b/collections/nemo_asr/nemo_asr/data_layer.py @@ -13,9 +13,9 @@ from .parts.dataset import ( AudioDataset, - seq_collate_fn, KaldiFeatureDataset, TranscriptDataset, + seq_collate_fn, ) from .parts.features import WaveformFeaturizer diff --git a/collections/nemo_asr/nemo_asr/greedy_ctc_decoder.py b/collections/nemo_asr/nemo_asr/greedy_ctc_decoder.py index 9ec4dcb08cc9c..2a829ed91b8ca 100644 --- a/collections/nemo_asr/nemo_asr/greedy_ctc_decoder.py +++ b/collections/nemo_asr/nemo_asr/greedy_ctc_decoder.py @@ -3,11 +3,11 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) diff --git a/collections/nemo_asr/nemo_asr/helpers.py b/collections/nemo_asr/nemo_asr/helpers.py index 272812f9eb589..1cd9aa87fd23a 100644 --- a/collections/nemo_asr/nemo_asr/helpers.py +++ b/collections/nemo_asr/nemo_asr/helpers.py @@ -1,6 +1,7 @@ # Copyright (c) 2019 NVIDIA Corporation import torch + import nemo from .metrics import word_error_rate diff --git a/collections/nemo_asr/nemo_asr/jasper.py b/collections/nemo_asr/nemo_asr/jasper.py index eb77d1cacf547..38d4066b56671 100644 --- a/collections/nemo_asr/nemo_asr/jasper.py +++ b/collections/nemo_asr/nemo_asr/jasper.py @@ -1,22 +1,23 @@ # Copyright (c) 2019 NVIDIA Corporation +from typing import Optional + import torch import torch.nn as nn import torch.nn.functional as F -from typing import Optional from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_types import ( - NeuralType, AxisType, - ChannelTag, BatchTag, - TimeTag, - SpectrogramSignalTag, - ProcessedTimeTag, + ChannelTag, EncodedRepresentationTag, + NeuralType, + ProcessedTimeTag, + SpectrogramSignalTag, + TimeTag, ) -from .parts.jasper import JasperBlock, jasper_activations, init_weights +from .parts.jasper import JasperBlock, init_weights, jasper_activations class JasperEncoder(TrainableNM): diff --git a/collections/nemo_asr/nemo_asr/las/helpers.py b/collections/nemo_asr/nemo_asr/las/helpers.py index 36154398ab274..16947223c651f 100644 --- a/collections/nemo_asr/nemo_asr/las/helpers.py +++ b/collections/nemo_asr/nemo_asr/las/helpers.py @@ -2,9 +2,9 @@ from pprint import pformat import torch + import nemo from nemo.backends.pytorch.common.metrics import char_lm_metrics - from nemo_asr.metrics import word_error_rate ENG_MWN = 5.3 diff --git a/collections/nemo_asr/nemo_asr/las/misc.py b/collections/nemo_asr/nemo_asr/las/misc.py index 54e9cf3f37b85..158893b275a94 100644 --- a/collections/nemo_asr/nemo_asr/las/misc.py +++ b/collections/nemo_asr/nemo_asr/las/misc.py @@ -4,11 +4,11 @@ from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) from nemo_asr.jasper import init_weights as jasper_init_weights diff --git a/collections/nemo_asr/nemo_asr/losses.py b/collections/nemo_asr/nemo_asr/losses.py index 5d9cb842ae5f5..d4e8ad17d2ed6 100644 --- a/collections/nemo_asr/nemo_asr/losses.py +++ b/collections/nemo_asr/nemo_asr/losses.py @@ -4,11 +4,11 @@ from nemo.backends.pytorch.nm import LossNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) diff --git a/collections/nemo_asr/nemo_asr/parts/cleaners.py b/collections/nemo_asr/nemo_asr/parts/cleaners.py index 3a1c3692d4078..200e10344f2f4 100755 --- a/collections/nemo_asr/nemo_asr/parts/cleaners.py +++ b/collections/nemo_asr/nemo_asr/parts/cleaners.py @@ -1,8 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation -import inflect import re +import inflect from unidecode import unidecode NUM_CHECK = re.compile(r'([$]?)(^|\s)(\S*[0-9]\S*)(?=(\s|$)((\S*)(\s|$))?)') diff --git a/collections/nemo_asr/nemo_asr/parts/collections.py b/collections/nemo_asr/nemo_asr/parts/collections.py index b2b38b2f47314..617ae95e9edb1 100644 --- a/collections/nemo_asr/nemo_asr/parts/collections.py +++ b/collections/nemo_asr/nemo_asr/parts/collections.py @@ -1,13 +1,12 @@ # Copyright (c) 2019 NVIDIA Corporation import collections import os -from typing import Optional, List, Union +from typing import List, Optional, Union import pandas as pd import nemo -from nemo_asr.parts import manifest -from nemo_asr.parts import parsers +from nemo_asr.parts import manifest, parsers class _Collection(collections.UserList): diff --git a/collections/nemo_asr/nemo_asr/parts/dataset.py b/collections/nemo_asr/nemo_asr/parts/dataset.py index c11928e32226f..66bffc16394b3 100644 --- a/collections/nemo_asr/nemo_asr/parts/dataset.py +++ b/collections/nemo_asr/nemo_asr/parts/dataset.py @@ -8,8 +8,7 @@ from torch.utils.data import Dataset import nemo -from nemo_asr.parts import collections -from nemo_asr.parts import parsers +from nemo_asr.parts import collections, parsers def seq_collate_fn(batch, token_pad_value=0): diff --git a/collections/nemo_asr/nemo_asr/parts/features.py b/collections/nemo_asr/nemo_asr/parts/features.py index 60e86c8fc1809..e90c221fe07f7 100644 --- a/collections/nemo_asr/nemo_asr/parts/features.py +++ b/collections/nemo_asr/nemo_asr/parts/features.py @@ -1,15 +1,17 @@ # Taken straight from Patter https://github.com/ryanleary/patter # TODO: review, and copyright and fix/add comments import math + import librosa import torch import torch.nn as nn -from .perturb import AudioAugmentor -from .segment import AudioSegment from torch_stft import STFT import nemo +from .perturb import AudioAugmentor +from .segment import AudioSegment + CONSTANT = 1e-5 diff --git a/collections/nemo_asr/nemo_asr/parts/jasper.py b/collections/nemo_asr/nemo_asr/parts/jasper.py index 7d1c65624b859..d570cbe8a4d83 100644 --- a/collections/nemo_asr/nemo_asr/parts/jasper.py +++ b/collections/nemo_asr/nemo_asr/parts/jasper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Optional, List +from typing import List, Optional, Tuple import torch import torch.nn as nn diff --git a/collections/nemo_asr/nemo_asr/parts/manifest.py b/collections/nemo_asr/nemo_asr/parts/manifest.py index 665494c15217e..f7e86a25eab8c 100644 --- a/collections/nemo_asr/nemo_asr/parts/manifest.py +++ b/collections/nemo_asr/nemo_asr/parts/manifest.py @@ -1,6 +1,6 @@ # Copyright (c) 2019 NVIDIA Corporation import json -from typing import Union, Iterator, Dict, Any, List +from typing import Any, Dict, Iterator, List, Union class ManifestBase: diff --git a/collections/nemo_asr/nemo_asr/parts/parsers.py b/collections/nemo_asr/nemo_asr/parts/parsers.py index b12bc809a656d..89b511cffdbf7 100644 --- a/collections/nemo_asr/nemo_asr/parts/parsers.py +++ b/collections/nemo_asr/nemo_asr/parts/parsers.py @@ -1,8 +1,9 @@ # Copyright (c) 2019 NVIDIA Corporation import string -from typing import Optional, List +from typing import List, Optional import frozendict + from nemo_asr.parts import cleaners diff --git a/collections/nemo_asr/nemo_asr/parts/perturb.py b/collections/nemo_asr/nemo_asr/parts/perturb.py index ead4bc7afe091..5724b5afd709b 100644 --- a/collections/nemo_asr/nemo_asr/parts/perturb.py +++ b/collections/nemo_asr/nemo_asr/parts/perturb.py @@ -5,8 +5,7 @@ import librosa from scipy import signal -from nemo_asr.parts import collections -from nemo_asr.parts import parsers +from nemo_asr.parts import collections, parsers from .segment import AudioSegment diff --git a/collections/nemo_nlp/nemo_nlp/__init__.py b/collections/nemo_nlp/nemo_nlp/__init__.py index fd39d366c7533..d7ce636285213 100644 --- a/collections/nemo_nlp/nemo_nlp/__init__.py +++ b/collections/nemo_nlp/nemo_nlp/__init__.py @@ -13,15 +13,13 @@ # limitations under the License. # ============================================================================= +import nemo + from .data import * from .huggingface import * from .modules import * from .transformer import * - -import nemo - - name = "nemo_nlp" backend = nemo.core.Backend.PyTorch __version__ = "0.9.0" diff --git a/collections/nemo_nlp/nemo_nlp/data/data_layers.py b/collections/nemo_nlp/nemo_nlp/data/data_layers.py index 4920cdc806302..ccf6c6dbc3d20 100644 --- a/collections/nemo_nlp/nemo_nlp/data/data_layers.py +++ b/collections/nemo_nlp/nemo_nlp/data/data_layers.py @@ -30,11 +30,12 @@ import torch from torch.utils import data as pt_data -from .datasets import * import nemo from nemo.backends.pytorch.nm import DataLayerNM from nemo.core.neural_types import * +from .datasets import * + class TextDataLayer(DataLayerNM): """ diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/bert_pretraining.py b/collections/nemo_nlp/nemo_nlp/data/datasets/bert_pretraining.py index 18a8fef59b973..ee220fe70b13b 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/bert_pretraining.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/bert_pretraining.py @@ -20,6 +20,7 @@ import os import pickle import random + import h5py import numpy as np from torch.utils.data import Dataset diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/glue.py b/collections/nemo_nlp/nemo_nlp/data/datasets/glue.py index 4b579b01a4326..7d1fe06c0cead 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/glue.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/glue.py @@ -20,10 +20,11 @@ https://github.com/huggingface/transformers """ -import nemo import numpy as np from torch.utils.data import Dataset +import nemo + class GLUEDataset(Dataset): def __init__( diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/punctuation_capitalization.py b/collections/nemo_nlp/nemo_nlp/data/datasets/punctuation_capitalization.py index aafb044e8b1e4..4f7422dd4c0c7 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/punctuation_capitalization.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/punctuation_capitalization.py @@ -24,10 +24,11 @@ import pickle import random -import nemo import numpy as np from torch.utils.data import Dataset +import nemo + from . import utils diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/sentence_classification.py b/collections/nemo_nlp/nemo_nlp/data/datasets/sentence_classification.py index 7ba9e662a43f4..2045dad4851d3 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/sentence_classification.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/sentence_classification.py @@ -22,10 +22,11 @@ import random -import nemo import numpy as np from torch.utils.data import Dataset +import nemo + from . import utils diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/squad.py b/collections/nemo_nlp/nemo_nlp/data/datasets/squad.py index fec0478521ef3..2eba06afffda2 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/squad.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/squad.py @@ -22,25 +22,26 @@ import sys import numpy as np -from tqdm import tqdm import torch from torch.utils.data import Dataset +from tqdm import tqdm import nemo from nemo_nlp.utils.nlp_utils import _is_whitespace -from .utils import DataProcessor + from ...utils.metrics.squad_metrics import ( _compute_softmax, _get_best_indexes, apply_no_ans_threshold, exact_match_score, - make_eval_dict, f1_score, + find_all_best_thresh, get_final_text, - normalize_answer, + make_eval_dict, merge_eval, - find_all_best_thresh, + normalize_answer, ) +from .utils import DataProcessor """ diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/token_classification.py b/collections/nemo_nlp/nemo_nlp/data/datasets/token_classification.py index 3dd99032cb50d..f3414476bb6ed 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/token_classification.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/token_classification.py @@ -24,10 +24,11 @@ import pickle import random -import nemo import numpy as np from torch.utils.data import Dataset +import nemo + from . import utils diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/translation.py b/collections/nemo_nlp/nemo_nlp/data/datasets/translation.py index 6d4a043857abf..aef227b7c0cec 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/translation.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/translation.py @@ -18,7 +18,7 @@ import numpy as np from torch.utils.data import Dataset -from ..utils import dataset_to_ids, clean_src_and_target +from ..utils import clean_src_and_target, dataset_to_ids class TranslationDataset(Dataset): diff --git a/collections/nemo_nlp/nemo_nlp/data/datasets/utils.py b/collections/nemo_nlp/nemo_nlp/data/datasets/utils.py index 4ec45f6cdee33..fdc6ff252be97 100644 --- a/collections/nemo_nlp/nemo_nlp/data/datasets/utils.py +++ b/collections/nemo_nlp/nemo_nlp/data/datasets/utils.py @@ -1,4 +1,3 @@ -from collections import Counter import csv import glob import itertools @@ -8,20 +7,21 @@ import re import shutil import subprocess +from collections import Counter -import nemo import numpy as np from sentencepiece import SentencePieceTrainer as SPT from tqdm import tqdm +import nemo + from ...utils.nlp_utils import ( get_vocab, + label2idx, write_vocab, write_vocab_in_order, - label2idx, ) - DATABASE_EXISTS_TMP = '{} dataset has already been processed and stored at {}' MODE_EXISTS_TMP = ( '{} mode of {} dataset has already been processed and stored at {}' diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/__init__.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/__init__.py index 41c6646c77e62..ba9baba6c89c4 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/__init__.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/__init__.py @@ -1,6 +1,6 @@ -from .spc_tokenizer import SentencePieceTokenizer from .bert_tokenizer import NemoBertTokenizer -from .yttm_tokenizer import YouTokenToMeTokenizer +from .char_tokenizer import CharTokenizer from .gpt2_tokenizer import NemoGPT2Tokenizer +from .spc_tokenizer import SentencePieceTokenizer from .word_tokenizer import WordTokenizer -from .char_tokenizer import CharTokenizer +from .yttm_tokenizer import YouTokenToMeTokenizer diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/bert_tokenizer.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/bert_tokenizer.py index c1844cd514517..c564a4c6a4dcc 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/bert_tokenizer.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/bert_tokenizer.py @@ -1,7 +1,9 @@ -from .tokenizer_spec import TokenizerSpec -from transformers import BertTokenizer import re +from transformers import BertTokenizer + +from .tokenizer_spec import TokenizerSpec + def handle_quotes(text): text_ = "" diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/gpt2_tokenizer.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/gpt2_tokenizer.py index 6b809c629e74a..7c7417c9f0c7f 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/gpt2_tokenizer.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/gpt2_tokenizer.py @@ -1,6 +1,7 @@ -from .tokenizer_spec import TokenizerSpec from transformers import GPT2Tokenizer +from .tokenizer_spec import TokenizerSpec + class NemoGPT2Tokenizer(TokenizerSpec): def __init__( diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/spc_tokenizer.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/spc_tokenizer.py index 7dc21f1efe9a7..67a2c00bda3e0 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/spc_tokenizer.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/spc_tokenizer.py @@ -1,4 +1,5 @@ import sentencepiece as spm + from .tokenizer_spec import TokenizerSpec diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/tokenizer_spec.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/tokenizer_spec.py index 687df89b26504..eeadf617c1896 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/tokenizer_spec.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/tokenizer_spec.py @@ -1,4 +1,4 @@ -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from typing import List diff --git a/collections/nemo_nlp/nemo_nlp/data/tokenizers/yttm_tokenizer.py b/collections/nemo_nlp/nemo_nlp/data/tokenizers/yttm_tokenizer.py index 21bdd9debc9a7..3f0b84e1f2928 100644 --- a/collections/nemo_nlp/nemo_nlp/data/tokenizers/yttm_tokenizer.py +++ b/collections/nemo_nlp/nemo_nlp/data/tokenizers/yttm_tokenizer.py @@ -1,4 +1,5 @@ import youtokentome as yttm + from .tokenizer_spec import TokenizerSpec diff --git a/collections/nemo_nlp/nemo_nlp/data/utils.py b/collections/nemo_nlp/nemo_nlp/data/utils.py index 5854a5daba26d..91565f7de0142 100644 --- a/collections/nemo_nlp/nemo_nlp/data/utils.py +++ b/collections/nemo_nlp/nemo_nlp/data/utils.py @@ -3,9 +3,10 @@ import re import string -import nemo import numpy as np +import nemo + def dataset_to_ids(dataset, tokenizer, cache_ids=False, add_bos_eos=True): """ diff --git a/collections/nemo_nlp/nemo_nlp/huggingface/bert.py b/collections/nemo_nlp/nemo_nlp/huggingface/bert.py index 0c030b2b25351..3834678686d65 100644 --- a/collections/nemo_nlp/nemo_nlp/huggingface/bert.py +++ b/collections/nemo_nlp/nemo_nlp/huggingface/bert.py @@ -1,11 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation -from typing import Optional, List +from typing import List, Optional from transformers import ( + BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BertConfig, BertModel, - BERT_PRETRAINED_MODEL_ARCHIVE_MAP, - BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ) from nemo.backends.pytorch.nm import TrainableNM diff --git a/collections/nemo_nlp/nemo_nlp/modules/classifiers.py b/collections/nemo_nlp/nemo_nlp/modules/classifiers.py index 4d7f620967c49..43869a2d0ac61 100644 --- a/collections/nemo_nlp/nemo_nlp/modules/classifiers.py +++ b/collections/nemo_nlp/nemo_nlp/modules/classifiers.py @@ -9,13 +9,12 @@ import torch.nn as nn from nemo.backends.pytorch.common import MultiLayerPerceptron -from nemo.backends.pytorch.nm import TrainableNM, LossNM +from nemo.backends.pytorch.nm import LossNM, TrainableNM from nemo.core.neural_types import * from nemo_nlp.transformer.utils import gelu from ..transformer.utils import transformer_weights_init - ACT2FN = {"gelu": gelu, "relu": nn.functional.relu} diff --git a/collections/nemo_nlp/nemo_nlp/modules/losses.py b/collections/nemo_nlp/nemo_nlp/modules/losses.py index 37125136e52e6..4ce15a60eb5d9 100644 --- a/collections/nemo_nlp/nemo_nlp/modules/losses.py +++ b/collections/nemo_nlp/nemo_nlp/modules/losses.py @@ -1,12 +1,11 @@ import torch from torch import nn - from nemo.backends.pytorch.nm import LossNM from nemo.core.neural_types import * -from .pytorch_utils import SmoothedCrossEntropyLoss -from ..utils.nlp_utils import mask_padded_tokens +from ..utils.nlp_utils import mask_padded_tokens +from .pytorch_utils import SmoothedCrossEntropyLoss __all__ = [ 'JointIntentSlotLoss', diff --git a/collections/nemo_nlp/nemo_nlp/modules/transformer_nm.py b/collections/nemo_nlp/nemo_nlp/modules/transformer_nm.py index 664b02c963b15..333f5ccfd4a45 100644 --- a/collections/nemo_nlp/nemo_nlp/modules/transformer_nm.py +++ b/collections/nemo_nlp/nemo_nlp/modules/transformer_nm.py @@ -11,15 +11,15 @@ import math -from nemo.backends.pytorch.nm import TrainableNM, LossNM +from nemo.backends.pytorch.nm import LossNM, TrainableNM from nemo.core.neural_types import * from ..transformer import ( + BeamSearchSequenceGenerator, + GreedySequenceGenerator, + TransformerDecoder, TransformerEmbedding, TransformerEncoder, - TransformerDecoder, - GreedySequenceGenerator, - BeamSearchSequenceGenerator, ) from ..transformer.utils import transformer_weights_init diff --git a/collections/nemo_nlp/nemo_nlp/transformer/generators.py b/collections/nemo_nlp/nemo_nlp/transformer/generators.py index fdaa951cbfadd..6da83de8f41be 100644 --- a/collections/nemo_nlp/nemo_nlp/transformer/generators.py +++ b/collections/nemo_nlp/nemo_nlp/transformer/generators.py @@ -7,8 +7,8 @@ import torch import torch.nn as nn -from .utils import NEG_INF from ..utils.nlp_utils import mask_padded_tokens +from .utils import NEG_INF class GreedySequenceGenerator(nn.Module): diff --git a/collections/nemo_nlp/nemo_nlp/transformer/modules.py b/collections/nemo_nlp/nemo_nlp/transformer/modules.py index 6f8b78518b25e..7a4f799eaad36 100644 --- a/collections/nemo_nlp/nemo_nlp/transformer/modules.py +++ b/collections/nemo_nlp/nemo_nlp/transformer/modules.py @@ -33,6 +33,11 @@ import math +import torch +from torch import nn + +from .utils import gelu + try: from apex.normalization import FusedLayerNorm except (AttributeError, ModuleNotFoundError): @@ -43,11 +48,6 @@ ) from torch.nn import LayerNorm as FusedLayerNorm -import torch -from torch import nn - -from .utils import gelu - class FixedPositionalEncoding(nn.Module): """ diff --git a/collections/nemo_nlp/nemo_nlp/utils/__init__.py b/collections/nemo_nlp/nemo_nlp/utils/__init__.py index c7abd0235d7e0..894348fc31148 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/__init__.py +++ b/collections/nemo_nlp/nemo_nlp/utils/__init__.py @@ -1,3 +1 @@ -from . import nlp_utils -from . import callbacks -from . import metrics +from . import callbacks, metrics, nlp_utils diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/bert_pretraining.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/bert_pretraining.py index f5e41a7462cc8..bc1b628bf4068 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/bert_pretraining.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/bert_pretraining.py @@ -1,9 +1,10 @@ # Copyright (c) 2019 NVIDIA Corporation __all__ = ['eval_iter_callback', 'eval_epochs_done_callback'] -import nemo import numpy as np +import nemo + def eval_iter_callback(tensors, global_vars): if "dev_mlm_loss" not in global_vars.keys(): diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/glue.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/glue.py index 6337d9179b676..3edb95fe6ea9d 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/glue.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/glue.py @@ -24,10 +24,11 @@ import os import random -import nemo import numpy as np from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import matthews_corrcoef, f1_score +from sklearn.metrics import f1_score, matthews_corrcoef + +import nemo def eval_iter_callback(tensors, global_vars): diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/joint_intent_slot.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/joint_intent_slot.py index 4e1b9c85f5f3b..aad4166b9e0f6 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/joint_intent_slot.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/joint_intent_slot.py @@ -5,10 +5,11 @@ import time import matplotlib +import numpy as np from matplotlib import pyplot as plt +from sklearn.metrics import classification_report, confusion_matrix + import nemo -import numpy as np -from sklearn.metrics import confusion_matrix, classification_report __all__ = ['eval_iter_callback', 'eval_epochs_done_callback'] diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/language_modeling.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/language_modeling.py index 48ca507482f75..daffe2c64d2d2 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/language_modeling.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/language_modeling.py @@ -1,9 +1,10 @@ # Copyright (c) 2019 NVIDIA Corporation __all__ = ['eval_iter_callback', 'eval_epochs_done_callback'] -import nemo import numpy as np +import nemo + GLOBAL_KEYS = ["eval_loss", "sys"] diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/punctuation_capitalization.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/punctuation_capitalization.py index b37f37dbecbed..c821216a9c95e 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/punctuation_capitalization.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/punctuation_capitalization.py @@ -3,10 +3,10 @@ import random -import nemo import numpy as np from sklearn.metrics import classification_report +import nemo from nemo_nlp.data.datasets.utils import list2str, tensor2list from nemo_nlp.utils.nlp_utils import plot_confusion_matrix diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/sentence_classification.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/sentence_classification.py index c13057c23ae13..4810bab9dde1e 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/sentence_classification.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/sentence_classification.py @@ -5,9 +5,9 @@ import random import time -from matplotlib import pyplot as plt # nopep8 import numpy as np # nopep8 -from sklearn.metrics import confusion_matrix, classification_report # nopep8 +from matplotlib import pyplot as plt # nopep8 +from sklearn.metrics import classification_report, confusion_matrix # nopep8 import nemo diff --git a/collections/nemo_nlp/nemo_nlp/utils/callbacks/translation.py b/collections/nemo_nlp/nemo_nlp/utils/callbacks/translation.py index a724740c1d01e..51dfd843767b6 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/callbacks/translation.py +++ b/collections/nemo_nlp/nemo_nlp/utils/callbacks/translation.py @@ -7,7 +7,6 @@ from ..metrics.sacrebleu import corpus_bleu - GLOBAL_KEYS = ["eval_loss", "ref", "sys", "sent_ids", "nonpad_tokens"] diff --git a/collections/nemo_nlp/nemo_nlp/utils/metrics/fairseq_tokenizer.py b/collections/nemo_nlp/nemo_nlp/utils/metrics/fairseq_tokenizer.py index e24b0bc2c6a52..274fb51afe71a 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/metrics/fairseq_tokenizer.py +++ b/collections/nemo_nlp/nemo_nlp/utils/metrics/fairseq_tokenizer.py @@ -3,8 +3,8 @@ master/PyTorch/Translation/Transformer/fairseq/tokenizer.py """ -import sys import re +import sys import unicodedata from collections import defaultdict diff --git a/collections/nemo_nlp/nemo_nlp/utils/metrics/sacrebleu.py b/collections/nemo_nlp/nemo_nlp/utils/metrics/sacrebleu.py index 56bc719c3fb27..bfb0493124f11 100755 --- a/collections/nemo_nlp/nemo_nlp/utils/metrics/sacrebleu.py +++ b/collections/nemo_nlp/nemo_nlp/utils/metrics/sacrebleu.py @@ -28,18 +28,17 @@ import math import os import re -import sys import ssl +import sys import unicodedata import urllib.request - from collections import Counter, namedtuple from itertools import zip_longest -from typing import List, Iterable, Tuple, Union +from typing import Iterable, List, Tuple, Union from nemo import logging -from .fairseq_tokenizer import tokenize_en +from .fairseq_tokenizer import tokenize_en VERSION = '1.3.5' diff --git a/collections/nemo_nlp/nemo_nlp/utils/nlp_utils.py b/collections/nemo_nlp/nemo_nlp/utils/nlp_utils.py index 52ec819cd5930..97bad93046763 100644 --- a/collections/nemo_nlp/nemo_nlp/utils/nlp_utils.py +++ b/collections/nemo_nlp/nemo_nlp/utils/nlp_utils.py @@ -1,11 +1,12 @@ import os import time -from matplotlib import pyplot as plt -import nemo import numpy as np +from matplotlib import pyplot as plt from sklearn.metrics import confusion_matrix +import nemo + def _is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: diff --git a/collections/nemo_simple_gan/nemo_simple_gan/__init__.py b/collections/nemo_simple_gan/nemo_simple_gan/__init__.py index 669439f9c2c72..4d8602ecfff9b 100644 --- a/collections/nemo_simple_gan/nemo_simple_gan/__init__.py +++ b/collections/nemo_simple_gan/nemo_simple_gan/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= -from .gan import * - from nemo.core import Backend +from .gan import * + name = "nemo_simple_gan" backend = Backend.PyTorch __version__ = "0.9.0" diff --git a/collections/nemo_simple_gan/nemo_simple_gan/gan.py b/collections/nemo_simple_gan/nemo_simple_gan/gan.py index 22469dc31f314..3364991c8f444 100644 --- a/collections/nemo_simple_gan/nemo_simple_gan/gan.py +++ b/collections/nemo_simple_gan/nemo_simple_gan/gan.py @@ -2,22 +2,22 @@ """A collection of Neural Modules to be used for training a WGAN-GP on MNIST""" import torch from torch.utils.data import Dataset -from torchvision import transforms, datasets +from torchvision import datasets, transforms from nemo.backends.pytorch.nm import ( - TrainableNM, - NonTrainableNM, - LossNM, DataLayerNM, + LossNM, + NonTrainableNM, + TrainableNM, ) from nemo.core import ( - NeuralType, + AxisType, BatchTag, ChannelTag, + DeviceType, HeightTag, + NeuralType, WidthTag, - AxisType, - DeviceType, ) diff --git a/collections/nemo_tts/nemo_tts/__init__.py b/collections/nemo_tts/nemo_tts/__init__.py index b7327d8e56c83..ce5a3d4c1dd05 100644 --- a/collections/nemo_tts/nemo_tts/__init__.py +++ b/collections/nemo_tts/nemo_tts/__init__.py @@ -14,26 +14,26 @@ # ============================================================================= from nemo.core import Backend +from .data_layers import AudioDataLayer +from .parts.helpers import ( + tacotron2_eval_log_to_tb_func, + tacotron2_log_to_tb_func, + tacotron2_process_eval_batch, + tacotron2_process_final_eval, + waveglow_eval_log_to_tb_func, + waveglow_log_to_tb_func, + waveglow_process_eval_batch, +) from .tacotron2_modules import ( MakeGate, - Tacotron2Loss, - Tacotron2Postnet, Tacotron2Decoder, Tacotron2DecoderInfer, Tacotron2Encoder, + Tacotron2Loss, + Tacotron2Postnet, TextEmbedding, ) -from .waveglow_modules import WaveGlowNM, WaveGlowInferNM, WaveGlowLoss -from .data_layers import AudioDataLayer -from .parts.helpers import ( - waveglow_log_to_tb_func, - waveglow_process_eval_batch, - waveglow_eval_log_to_tb_func, - tacotron2_log_to_tb_func, - tacotron2_process_eval_batch, - tacotron2_process_final_eval, - tacotron2_eval_log_to_tb_func, -) +from .waveglow_modules import WaveGlowInferNM, WaveGlowLoss, WaveGlowNM name = "nemo_tts" backend = Backend.PyTorch diff --git a/collections/nemo_tts/nemo_tts/data_layers.py b/collections/nemo_tts/nemo_tts/data_layers.py index 820aadf17308d..f63ecf471cb32 100644 --- a/collections/nemo_tts/nemo_tts/data_layers.py +++ b/collections/nemo_tts/nemo_tts/data_layers.py @@ -1,9 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation import torch + import nemo from nemo.backends.pytorch.nm import DataLayerNM from nemo.core import DeviceType from nemo.core.neural_types import * + from .parts.datasets import AudioOnlyDataset diff --git a/collections/nemo_tts/nemo_tts/parts/__init__.py b/collections/nemo_tts/nemo_tts/parts/__init__.py index 9080627cb54c1..1350837aada6a 100644 --- a/collections/nemo_tts/nemo_tts/parts/__init__.py +++ b/collections/nemo_tts/nemo_tts/parts/__init__.py @@ -1,16 +1,16 @@ from .datasets import AudioOnlyDataset -from .layers import get_mask_from_lengths -from .tacotron2 import Encoder, Decoder, Postnet -from .waveglow import WaveGlow from .helpers import ( - waveglow_log_to_tb_func, - waveglow_process_eval_batch, - waveglow_eval_log_to_tb_func, + tacotron2_eval_log_to_tb_func, tacotron2_log_to_tb_func, tacotron2_process_eval_batch, tacotron2_process_final_eval, - tacotron2_eval_log_to_tb_func, + waveglow_eval_log_to_tb_func, + waveglow_log_to_tb_func, + waveglow_process_eval_batch, ) +from .layers import get_mask_from_lengths +from .tacotron2 import Decoder, Encoder, Postnet +from .waveglow import WaveGlow __all__ = [ 'AudioOnlyDataset', diff --git a/collections/nemo_tts/nemo_tts/parts/datasets.py b/collections/nemo_tts/nemo_tts/parts/datasets.py index 46b375d23a79c..e7d379b18c3fd 100644 --- a/collections/nemo_tts/nemo_tts/parts/datasets.py +++ b/collections/nemo_tts/nemo_tts/parts/datasets.py @@ -2,8 +2,7 @@ import torch from torch.utils.data import Dataset -from nemo_asr.parts import collections -from nemo_asr.parts import parsers +from nemo_asr.parts import collections, parsers from nemo_asr.parts.segment import AudioSegment diff --git a/collections/nemo_tts/nemo_tts/parts/tacotron2.py b/collections/nemo_tts/nemo_tts/parts/tacotron2.py index 80d4361bbffd9..fe4f828643142 100644 --- a/collections/nemo_tts/nemo_tts/parts/tacotron2.py +++ b/collections/nemo_tts/nemo_tts/parts/tacotron2.py @@ -2,8 +2,8 @@ from math import sqrt import torch -from torch.autograd import Variable from torch import nn +from torch.autograd import Variable from torch.nn import functional as F from .layers import ConvNorm, LinearNorm, get_mask_from_lengths diff --git a/collections/nemo_tts/nemo_tts/parts/waveglow.py b/collections/nemo_tts/nemo_tts/parts/waveglow.py index 6b35391df2f67..becff2dca6cf3 100644 --- a/collections/nemo_tts/nemo_tts/parts/waveglow.py +++ b/collections/nemo_tts/nemo_tts/parts/waveglow.py @@ -1,7 +1,7 @@ # Copyright (c) 2019 NVIDIA Corporation import torch -from torch.autograd import Variable import torch.nn.functional as F +from torch.autograd import Variable @torch.jit.script diff --git a/collections/nemo_tts/nemo_tts/tacotron2_modules.py b/collections/nemo_tts/nemo_tts/tacotron2_modules.py index e820af4147b30..7b1197bdeb611 100644 --- a/collections/nemo_tts/nemo_tts/tacotron2_modules.py +++ b/collections/nemo_tts/nemo_tts/tacotron2_modules.py @@ -5,10 +5,11 @@ from torch import nn from torch.nn.functional import pad -from nemo.backends.pytorch.nm import TrainableNM, NonTrainableNM, LossNM +from nemo.backends.pytorch.nm import LossNM, NonTrainableNM, TrainableNM from nemo.core.neural_types import * -from .parts.tacotron2 import Encoder, Decoder, Postnet + from .parts.layers import get_mask_from_lengths +from .parts.tacotron2 import Decoder, Encoder, Postnet class TextEmbedding(TrainableNM): diff --git a/collections/nemo_tts/nemo_tts/waveglow_modules.py b/collections/nemo_tts/nemo_tts/waveglow_modules.py index 03e89ccbe7e96..b82412a4c9233 100644 --- a/collections/nemo_tts/nemo_tts/waveglow_modules.py +++ b/collections/nemo_tts/nemo_tts/waveglow_modules.py @@ -3,8 +3,9 @@ import numpy as np import torch -from nemo.backends.pytorch.nm import TrainableNM, LossNM +from nemo.backends.pytorch.nm import LossNM, TrainableNM from nemo.core.neural_types import * + from .parts.waveglow import WaveGlow diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py index ad281c8d42091..4cbc1481fc6a3 100644 --- a/docs/docs_zh/sources/source/conf.py +++ b/docs/docs_zh/sources/source/conf.py @@ -20,6 +20,7 @@ import os import sys from unittest.mock import MagicMock + import nemo sys.path.insert(0, os.path.abspath(".")) diff --git a/docs/sources/source/conf.py b/docs/sources/source/conf.py index fb9bf454134c8..ca3c55d018ea4 100644 --- a/docs/sources/source/conf.py +++ b/docs/sources/source/conf.py @@ -13,14 +13,15 @@ # All configuration values have a default; values that are commented out # infer to show the default. +import os +import sys +from unittest.mock import MagicMock + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import nemo -import os -import sys -from unittest.mock import MagicMock sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) diff --git a/examples/applications/asr_service/app/__init__.py b/examples/applications/asr_service/app/__init__.py index b6ce3f84d1ca7..6913bc510225b 100644 --- a/examples/applications/asr_service/app/__init__.py +++ b/examples/applications/asr_service/app/__init__.py @@ -1,10 +1,12 @@ # Copyright (c) 2019 NVIDIA Corporation import os -from flask import Flask + from ruamel.yaml import YAML import nemo import nemo_asr +from app import routes # noqa +from flask import Flask app = Flask(__name__) # make sure WORK_DIR exists before calling your service @@ -60,7 +62,6 @@ else: print("Beam search is not enabled") -from app import routes # noqa if __name__ == '__main__': app.run() diff --git a/examples/applications/asr_service/app/routes.py b/examples/applications/asr_service/app/routes.py index b2affe2afe87c..01f69bd32b96b 100644 --- a/examples/applications/asr_service/app/routes.py +++ b/examples/applications/asr_service/app/routes.py @@ -3,28 +3,27 @@ import os import time -from flask import request -from werkzeug.utils import secure_filename - +import nemo +import nemo_asr from app import ( + ENABLE_NGRAM, + MODEL_YAML, + WORK_DIR, app, data_preprocessor, - jasper_encoder, - jasper_decoder, greedy_decoder, + jasper_decoder, + jasper_encoder, neural_factory, - MODEL_YAML, - WORK_DIR, - ENABLE_NGRAM, ) +from flask import request +from werkzeug.utils import secure_filename try: from app import beam_search_with_lm except ImportError: print("Not using Beam Search Decoder with LM") ENABLE_NGRAM = False -import nemo -import nemo_asr def wav_to_text(manifest, greedy=True): diff --git a/examples/asr/experimental/garnet.py b/examples/asr/experimental/garnet.py index f1e16e12057cc..c28c3d7070b56 100644 --- a/examples/asr/experimental/garnet.py +++ b/examples/asr/experimental/garnet.py @@ -13,11 +13,11 @@ from tensorboardX import SummaryWriter import nemo -from nemo.core.callbacks import ValueSetterCallback, UnfreezeCallback import nemo.utils.argparse as nm_argparse +import nemo_asr +from nemo.core.callbacks import UnfreezeCallback, ValueSetterCallback from nemo.utils.lr_policies import SquareAnnealing from nemo.utils.misc import Config -import nemo_asr from nemo_asr.las.helpers import ( process_evaluation_batch, process_evaluation_epoch, diff --git a/examples/asr/experimental/garnet_rnnlm.py b/examples/asr/experimental/garnet_rnnlm.py index 3baa6db14ce20..a0ad8488bd3f7 100644 --- a/examples/asr/experimental/garnet_rnnlm.py +++ b/examples/asr/experimental/garnet_rnnlm.py @@ -13,11 +13,11 @@ from tensorboardX import SummaryWriter import nemo +import nemo.utils.argparse as nm_argparse +import nemo_asr from nemo.core.callbacks import ValueSetterCallback from nemo.utils.lr_policies import SquareAnnealing -import nemo.utils.argparse as nm_argparse from nemo.utils.misc import Config -import nemo_asr from nemo_asr.las.helpers import ( process_evaluation_batch, process_evaluation_epoch, diff --git a/examples/asr/jasper.py b/examples/asr/jasper.py index e0030d57fc185..104b22b670aee 100644 --- a/examples/asr/jasper.py +++ b/examples/asr/jasper.py @@ -1,16 +1,16 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import math import os +from functools import partial from ruamel.yaml import YAML import nemo -from nemo.utils.lr_policies import CosineAnnealing import nemo.utils.argparse as nm_argparse import nemo_asr +from nemo.utils.lr_policies import CosineAnnealing from nemo_asr.helpers import ( monitor_asr_train_progress, process_evaluation_batch, diff --git a/examples/asr/jasper_aishell.py b/examples/asr/jasper_aishell.py index 85e781beed5d4..798f509f96d35 100644 --- a/examples/asr/jasper_aishell.py +++ b/examples/asr/jasper_aishell.py @@ -1,15 +1,15 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import os +from functools import partial from ruamel.yaml import YAML import nemo -from nemo.utils.lr_policies import SquareAnnealing import nemo.utils.argparse as nm_argparse import nemo_asr +from nemo.utils.lr_policies import SquareAnnealing from nemo_asr.helpers import ( monitor_asr_train_progress, process_evaluation_batch, diff --git a/examples/asr/jasper_aishell_infer.py b/examples/asr/jasper_aishell_infer.py index c4e08ee058da8..cbb858dc32dda 100644 --- a/examples/asr/jasper_aishell_infer.py +++ b/examples/asr/jasper_aishell_infer.py @@ -9,9 +9,9 @@ import nemo import nemo_asr from nemo_asr.helpers import ( - word_error_rate, post_process_predictions, post_process_transcripts, + word_error_rate, ) diff --git a/examples/asr/jasper_an4.py b/examples/asr/jasper_an4.py index 7541c8bb71aef..c4e4b27db8133 100644 --- a/examples/asr/jasper_an4.py +++ b/examples/asr/jasper_an4.py @@ -1,23 +1,23 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import math import os +from functools import partial from ruamel.yaml import YAML import nemo import nemo.utils.argparse as nm_argparse -from nemo.utils.lr_policies import CosineAnnealing import nemo_asr +from nemo.utils.lr_policies import CosineAnnealing from nemo_asr.helpers import ( monitor_asr_train_progress, + post_process_predictions, + post_process_transcripts, process_evaluation_batch, process_evaluation_epoch, word_error_rate, - post_process_predictions, - post_process_transcripts, ) diff --git a/examples/asr/jasper_eval.py b/examples/asr/jasper_eval.py index 42efc9cd42bab..4a2f23d9d9a7b 100644 --- a/examples/asr/jasper_eval.py +++ b/examples/asr/jasper_eval.py @@ -6,15 +6,15 @@ import os import pickle -from ruamel.yaml import YAML import numpy as np +from ruamel.yaml import YAML import nemo import nemo_asr from nemo_asr.helpers import ( - word_error_rate, post_process_predictions, post_process_transcripts, + word_error_rate, ) diff --git a/examples/asr/quartznet.py b/examples/asr/quartznet.py index 644f32a7257b9..62879703d101e 100644 --- a/examples/asr/quartznet.py +++ b/examples/asr/quartznet.py @@ -1,15 +1,15 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import os +from functools import partial from ruamel.yaml import YAML import nemo -from nemo.utils.lr_policies import CosineAnnealing import nemo.utils.argparse as nm_argparse import nemo_asr +from nemo.utils.lr_policies import CosineAnnealing from nemo_asr.helpers import ( monitor_asr_train_progress, process_evaluation_batch, diff --git a/examples/image/gan.py b/examples/image/gan.py index 49ab004d4d0f0..cdc8d47bdf5ed 100644 --- a/examples/image/gan.py +++ b/examples/image/gan.py @@ -7,13 +7,12 @@ from tensorboardX import SummaryWriter import nemo +import nemo_simple_gan from nemo.backends.pytorch.torchvision.helpers import ( - eval_iter_callback, - eval_epochs_done_callback, compute_accuracy, + eval_epochs_done_callback, + eval_iter_callback, ) -import nemo_simple_gan - parser = argparse.ArgumentParser(description='MNIST') parser.add_argument("--local_rank", default=None, type=int) diff --git a/examples/image/resnet50.py b/examples/image/resnet50.py index 4fa00930691f9..730b94dbfbca8 100644 --- a/examples/image/resnet50.py +++ b/examples/image/resnet50.py @@ -1,15 +1,17 @@ # Copyright (c) 2019 NVIDIA Corporation +import argparse +import os +import sys + from tensorboardX import SummaryWriter -from nemo.utils.lr_policies import SquareAnnealing + +import nemo from nemo.backends.pytorch.torchvision.helpers import ( - eval_iter_callback, - eval_epochs_done_callback, compute_accuracy, + eval_epochs_done_callback, + eval_iter_callback, ) -import nemo -import argparse -import os -import sys +from nemo.utils.lr_policies import SquareAnnealing sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) diff --git a/examples/image/transfer_learning.py b/examples/image/transfer_learning.py index f51c0552a03e6..69e03fd84c033 100644 --- a/examples/image/transfer_learning.py +++ b/examples/image/transfer_learning.py @@ -1,15 +1,17 @@ +import argparse +import os +import subprocess +import sys +import zipfile + from tensorboardX import SummaryWriter + import nemo from nemo.backends.pytorch.torchvision.helpers import ( - eval_iter_callback, - eval_epochs_done_callback, compute_accuracy, + eval_epochs_done_callback, + eval_iter_callback, ) -import argparse -import os -import subprocess -import zipfile -import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) diff --git a/examples/nlp/asr_postprocessor.py b/examples/nlp/asr_postprocessor.py index 7de79b237b371..0cd7ba50fe085 100644 --- a/examples/nlp/asr_postprocessor.py +++ b/examples/nlp/asr_postprocessor.py @@ -1,18 +1,17 @@ # Copyright (c) 2019 NVIDIA Corporation import math +import os import torch -import os import nemo +import nemo_nlp from nemo.core.callbacks import CheckpointCallback from nemo.utils.lr_policies import SquareAnnealing - -import nemo_nlp from nemo_nlp.data.tokenizers.bert_tokenizer import NemoBertTokenizer from nemo_nlp.utils.callbacks.translation import ( - eval_iter_callback, eval_epochs_done_callback_wer, + eval_iter_callback, ) parser = nemo.utils.NemoArgParser(description='ASR postprocessor') diff --git a/examples/nlp/bert_pretraining.py b/examples/nlp/bert_pretraining.py index a373e34d65008..218e5644f2b90 100644 --- a/examples/nlp/bert_pretraining.py +++ b/examples/nlp/bert_pretraining.py @@ -59,20 +59,21 @@ should finish under 5 days and yield an MRPC score of ACC/F1 85.05/89.35. """ import argparse -import os import math +import os + import torch + import nemo -from nemo.utils.lr_policies import get_lr_policy -from pytorch_transformers import BertConfig import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.data.datasets.utils import BERTPretrainingDataDesc from nemo_nlp.transformer.utils import gelu from nemo_nlp.utils.callbacks.bert_pretraining import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) - +from pytorch_transformers import BertConfig parser = argparse.ArgumentParser(description='BERT pretraining') parser.add_argument("--local_rank", default=None, type=int) diff --git a/examples/nlp/glue_with_BERT.py b/examples/nlp/glue_with_BERT.py index 53bcb39b63743..b934f4498ac58 100644 --- a/examples/nlp/glue_with_BERT.py +++ b/examples/nlp/glue_with_BERT.py @@ -61,23 +61,26 @@ """ import argparse +import json import os import sys import nemo +import nemo_nlp from nemo.backends.pytorch.common import CrossEntropyLoss, MSELoss from nemo.utils.lr_policies import get_lr_policy -import json -import nemo_nlp -from nemo_nlp import GlueDataLayerClassification, GlueDataLayerRegression -from nemo_nlp import NemoBertTokenizer, SentencePieceTokenizer +from nemo_nlp import ( + GlueDataLayerClassification, + GlueDataLayerRegression, + NemoBertTokenizer, + SentencePieceTokenizer, +) +from nemo_nlp.data.datasets.utils import output_modes, processors from nemo_nlp.utils.callbacks.glue import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) -from nemo_nlp.data.datasets.utils import processors, output_modes - parser = argparse.ArgumentParser(description="GLUE_with_pretrained_BERT") # Parsing arguments diff --git a/examples/nlp/joint_intent_slot_infer.py b/examples/nlp/joint_intent_slot_infer.py index afa97f0dad814..a7e788ce5a83d 100644 --- a/examples/nlp/joint_intent_slot_infer.py +++ b/examples/nlp/joint_intent_slot_infer.py @@ -2,14 +2,13 @@ import os import numpy as np +from sklearn.metrics import classification_report, confusion_matrix from transformers import BertTokenizer -from sklearn.metrics import confusion_matrix, classification_report import nemo import nemo_nlp from nemo_nlp.data.datasets.utils import JointIntentSlotDataDesc - # Parsing arguments parser = argparse.ArgumentParser(description='Joint-intent BERT') parser.add_argument("--local_rank", default=None, type=int) diff --git a/examples/nlp/joint_intent_slot_infer_b1.py b/examples/nlp/joint_intent_slot_infer_b1.py index 737e1ba94d860..c4cbfe1ef3822 100644 --- a/examples/nlp/joint_intent_slot_infer_b1.py +++ b/examples/nlp/joint_intent_slot_infer_b1.py @@ -8,7 +8,6 @@ from nemo_nlp.data.datasets.utils import JointIntentSlotDataDesc from nemo_nlp.utils.nlp_utils import read_intent_slot_outputs - # Parsing arguments parser = argparse.ArgumentParser(description='Joint-intent BERT') parser.add_argument("--max_seq_length", default=50, type=int) diff --git a/examples/nlp/joint_intent_slot_with_bert.py b/examples/nlp/joint_intent_slot_with_bert.py index b6f27182dbd63..60c4eef41d049 100644 --- a/examples/nlp/joint_intent_slot_with_bert.py +++ b/examples/nlp/joint_intent_slot_with_bert.py @@ -6,16 +6,14 @@ from transformers import BertTokenizer import nemo -from nemo.utils.lr_policies import get_lr_policy - import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.data.datasets.utils import JointIntentSlotDataDesc from nemo_nlp.utils.callbacks.joint_intent_slot import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) - # Parsing arguments parser = argparse.ArgumentParser( description='Joint intent slot filling system with pretrained BERT' diff --git a/examples/nlp/nmt_tutorial.py b/examples/nlp/nmt_tutorial.py index b74333ee90bae..efbc4a11703f7 100644 --- a/examples/nlp/nmt_tutorial.py +++ b/examples/nlp/nmt_tutorial.py @@ -6,12 +6,11 @@ import torch import nemo -from nemo.utils.lr_policies import get_lr_policy - import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.utils.callbacks.translation import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) parser = nemo.utils.NemoArgParser( diff --git a/examples/nlp/punctuation_capitalization.py b/examples/nlp/punctuation_capitalization.py index 32c8c3072e374..ac70193052be5 100644 --- a/examples/nlp/punctuation_capitalization.py +++ b/examples/nlp/punctuation_capitalization.py @@ -6,19 +6,18 @@ import sys import nemo -from nemo.utils.lr_policies import get_lr_policy - import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp import ( NemoBertTokenizer, SentencePieceTokenizer, - TokenClassifier, TokenClassificationLoss, + TokenClassifier, ) from nemo_nlp.data.datasets import utils from nemo_nlp.utils.callbacks.punctuation_capitalization import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) # Parsing arguments diff --git a/examples/nlp/scripts/process_wiki_zh.py b/examples/nlp/scripts/process_wiki_zh.py index c51932f91cb66..c3c7181c95ae7 100755 --- a/examples/nlp/scripts/process_wiki_zh.py +++ b/examples/nlp/scripts/process_wiki_zh.py @@ -15,12 +15,12 @@ # limitations under the License. # ============================================================================= -import os +import glob import json +import os +import re from concurrent.futures import ProcessPoolExecutor from functools import partial -import re -import glob def create_vocab( diff --git a/examples/nlp/sentence_classification_with_bert.py b/examples/nlp/sentence_classification_with_bert.py index 91c0bbae69746..070961c9686e8 100644 --- a/examples/nlp/sentence_classification_with_bert.py +++ b/examples/nlp/sentence_classification_with_bert.py @@ -2,18 +2,17 @@ import math import numpy as np -from transformers import BertTokenizer -from torch import nn import torch +from torch import nn +from transformers import BertTokenizer import nemo -from nemo.utils.lr_policies import get_lr_policy - import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.data.datasets.utils import SentenceClassificationDataDesc from nemo_nlp.utils.callbacks.sentence_classification import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) # Parsing arguments diff --git a/examples/nlp/squad.py b/examples/nlp/squad.py index 4590581c3246d..24f12b5c22364 100755 --- a/examples/nlp/squad.py +++ b/examples/nlp/squad.py @@ -66,8 +66,8 @@ import nemo_nlp from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.utils.callbacks.squad import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) diff --git a/examples/nlp/token_classification.py b/examples/nlp/token_classification.py index 962beb33e471b..6a7ae3208b92b 100644 --- a/examples/nlp/token_classification.py +++ b/examples/nlp/token_classification.py @@ -6,19 +6,18 @@ import sys import nemo -from nemo.utils.lr_policies import get_lr_policy - import nemo_nlp +from nemo.utils.lr_policies import get_lr_policy from nemo_nlp import ( NemoBertTokenizer, SentencePieceTokenizer, - TokenClassifier, TokenClassificationLoss, + TokenClassifier, ) from nemo_nlp.data.datasets import utils from nemo_nlp.utils.callbacks.token_classification import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) # Parsing arguments diff --git a/examples/nlp/transformer_lm.py b/examples/nlp/transformer_lm.py index 5b87107c4e0cc..97134f1778bac 100644 --- a/examples/nlp/transformer_lm.py +++ b/examples/nlp/transformer_lm.py @@ -2,16 +2,14 @@ import math import nemo -from nemo.utils.lr_policies import CosineAnnealing import nemo_nlp - +from nemo.utils.lr_policies import CosineAnnealing from nemo_nlp.data.datasets.utils import LanguageModelDataDesc from nemo_nlp.utils.callbacks.language_modeling import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) - parser = nemo.utils.NemoArgParser(description='LM Transformer') parser.set_defaults( train_dataset="train.txt", diff --git a/examples/start_here/chatbot_example.py b/examples/start_here/chatbot_example.py index 86719c35374af..fa3605d93b922 100644 --- a/examples/start_here/chatbot_example.py +++ b/examples/start_here/chatbot_example.py @@ -1,6 +1,7 @@ -import os import gzip +import os import shutil + import nemo # Get Data diff --git a/examples/start_here/chatbot_example2.py b/examples/start_here/chatbot_example2.py index 46f91c57f64b0..822c568ba513c 100644 --- a/examples/start_here/chatbot_example2.py +++ b/examples/start_here/chatbot_example2.py @@ -1,7 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation -import os import gzip +import os import shutil + import nemo # Get Data diff --git a/examples/tts/tacotron2.py b/examples/tts/tacotron2.py index 5adc173bb14f3..681d99bfd2732 100644 --- a/examples/tts/tacotron2.py +++ b/examples/tts/tacotron2.py @@ -1,22 +1,22 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import math import os +from functools import partial from ruamel.yaml import YAML import nemo -from nemo.utils.lr_policies import CosineAnnealing import nemo.utils.argparse as nm_argparse import nemo_asr import nemo_tts +from nemo.utils.lr_policies import CosineAnnealing from nemo_tts import ( + tacotron2_eval_log_to_tb_func, tacotron2_log_to_tb_func, tacotron2_process_eval_batch, tacotron2_process_final_eval, - tacotron2_eval_log_to_tb_func, ) diff --git a/examples/tts/waveglow.py b/examples/tts/waveglow.py index dccb135324ea1..4d40f7dbb5dea 100644 --- a/examples/tts/waveglow.py +++ b/examples/tts/waveglow.py @@ -1,8 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse import copy -from functools import partial import os +from functools import partial from ruamel.yaml import YAML @@ -11,9 +11,9 @@ import nemo_asr import nemo_tts from nemo_tts import ( + waveglow_eval_log_to_tb_func, waveglow_log_to_tb_func, waveglow_process_eval_batch, - waveglow_eval_log_to_tb_func, ) diff --git a/nemo/nemo/__init__.py b/nemo/nemo/__init__.py index ed63f759f04fd..0b54f7145f8d4 100644 --- a/nemo/nemo/__init__.py +++ b/nemo/nemo/__init__.py @@ -14,9 +14,7 @@ # ============================================================================= import logging -from . import backends -from . import core -from . import utils +from . import backends, core, utils from .backends.pytorch import tutorials __version__ = "0.9.0" diff --git a/nemo/nemo/backends/pytorch/__init__.py b/nemo/nemo/backends/pytorch/__init__.py index eb96fa45b69b8..05688901f6711 100644 --- a/nemo/nemo/backends/pytorch/__init__.py +++ b/nemo/nemo/backends/pytorch/__init__.py @@ -3,7 +3,7 @@ This package provides Neural Modules building blocks for building Software 2.0 projects """ -from . import tutorials, torchvision +from . import torchvision, tutorials from .actions import PtActions -from .nm import TrainableNM, NonTrainableNM, DataLayerNM, LossNM from .common import * +from .nm import DataLayerNM, LossNM, NonTrainableNM, TrainableNM diff --git a/nemo/nemo/backends/pytorch/actions.py b/nemo/nemo/backends/pytorch/actions.py index 2e52bfd945909..85bb61babf078 100644 --- a/nemo/nemo/backends/pytorch/actions.py +++ b/nemo/nemo/backends/pytorch/actions.py @@ -3,11 +3,12 @@ import itertools import json import os -from pathlib import Path -from typing import List, Optional, Dict # import onnx from collections import defaultdict +from pathlib import Path +from typing import Dict, List, Optional + import torch import torch.distributed as dist import torch.nn as nn @@ -15,18 +16,19 @@ import nemo from nemo.backends.pytorch.nm import TrainableNM -from .module_wrapper import TrainableNeuralModuleWrapper -from .nm import DataLayerNM -from .optimizers import Novograd, AdamW, master_params -from ...core import NmTensor, DeviceType, NeuralModule, DeploymentFormat -from ...core.neural_types import * + +from ...core import DeploymentFormat, DeviceType, NeuralModule, NmTensor from ...core.callbacks import ( ActionCallback, EvaluatorCallback, SimpleLossLoggerCallback, ) from ...core.neural_factory import Actions, ModelMode, Optimization +from ...core.neural_types import * from ...utils.helpers import get_checkpoint_from_dir +from .module_wrapper import TrainableNeuralModuleWrapper +from .nm import DataLayerNM +from .optimizers import AdamW, Novograd, master_params # these imports will happen on as-needed basis amp = None diff --git a/nemo/nemo/backends/pytorch/common/losses.py b/nemo/nemo/backends/pytorch/common/losses.py index 6045e00626490..d6440906852f2 100644 --- a/nemo/nemo/backends/pytorch/common/losses.py +++ b/nemo/nemo/backends/pytorch/common/losses.py @@ -3,12 +3,12 @@ from nemo.backends.pytorch.nm import LossNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, RegressionTag, + TimeTag, ) __all__ = ['SequenceLoss', 'CrossEntropyLoss', 'MSELoss'] diff --git a/nemo/nemo/backends/pytorch/common/other.py b/nemo/nemo/backends/pytorch/common/other.py index f4cf8be7766c9..fdfcc79c6f565 100644 --- a/nemo/nemo/backends/pytorch/common/other.py +++ b/nemo/nemo/backends/pytorch/common/other.py @@ -10,7 +10,7 @@ 'ZerosLikeNM', ] -from typing import Iterable, Optional, Mapping, Set, Dict +from typing import Dict, Iterable, Mapping, Optional, Set import torch import torch.nn as nn diff --git a/nemo/nemo/backends/pytorch/common/rnn.py b/nemo/nemo/backends/pytorch/common/rnn.py index db38aab774771..126c79e8196aa 100644 --- a/nemo/nemo/backends/pytorch/common/rnn.py +++ b/nemo/nemo/backends/pytorch/common/rnn.py @@ -11,11 +11,11 @@ from nemo.backends.pytorch.common.parts import Attention from nemo.backends.pytorch.nm import TrainableNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) from nemo.utils.misc import pad_to diff --git a/nemo/nemo/backends/pytorch/common/search.py b/nemo/nemo/backends/pytorch/common/search.py index 4f3fea6a605f2..6c31e13112805 100644 --- a/nemo/nemo/backends/pytorch/common/search.py +++ b/nemo/nemo/backends/pytorch/common/search.py @@ -4,11 +4,11 @@ from nemo.backends.pytorch.nm import NonTrainableNM from nemo.core.neural_types import ( - NeuralType, AxisType, BatchTag, - TimeTag, ChannelTag, + NeuralType, + TimeTag, ) INF = float('inf') diff --git a/nemo/nemo/backends/pytorch/common/zero_data.py b/nemo/nemo/backends/pytorch/common/zero_data.py index 582e9750f13ca..ad833b285cc8a 100644 --- a/nemo/nemo/backends/pytorch/common/zero_data.py +++ b/nemo/nemo/backends/pytorch/common/zero_data.py @@ -1,8 +1,8 @@ import torch from torch.utils.data import Dataset -from ...pytorch.nm import DataLayerNM from ....core.neural_types import * +from ...pytorch.nm import DataLayerNM def neuralType2TensorShape( diff --git a/nemo/nemo/backends/pytorch/module_wrapper.py b/nemo/nemo/backends/pytorch/module_wrapper.py index 02d781e481122..3de6252198d1e 100644 --- a/nemo/nemo/backends/pytorch/module_wrapper.py +++ b/nemo/nemo/backends/pytorch/module_wrapper.py @@ -2,7 +2,7 @@ import torch as t import torch.nn as nn -from ...core import NeuralModule, DeviceType +from ...core import DeviceType, NeuralModule from ...utils.helpers import rgetattr, rsetattr diff --git a/nemo/nemo/backends/pytorch/nm.py b/nemo/nemo/backends/pytorch/nm.py index a4213aede9ecf..b2ee322aa280b 100644 --- a/nemo/nemo/backends/pytorch/nm.py +++ b/nemo/nemo/backends/pytorch/nm.py @@ -1,12 +1,12 @@ # Copyright (c) 2019 NVIDIA Corporation from abc import abstractmethod -from typing import Dict, Set, Tuple, Optional, List +from typing import Dict, List, Optional, Set, Tuple import torch as t import torch.nn as nn -from ...core import NeuralModule, DeviceType, WeightShareTransform -from ...utils.helpers import rgetattr, rsetattr, get_cuda_device +from ...core import DeviceType, NeuralModule, WeightShareTransform +from ...utils.helpers import get_cuda_device, rgetattr, rsetattr class TrainableNM(NeuralModule, nn.Module): diff --git a/nemo/nemo/backends/pytorch/torchvision/data/image_folder.py b/nemo/nemo/backends/pytorch/torchvision/data/image_folder.py index 6ba0becb733c3..566bcd9534811 100644 --- a/nemo/nemo/backends/pytorch/torchvision/data/image_folder.py +++ b/nemo/nemo/backends/pytorch/torchvision/data/image_folder.py @@ -1,8 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation -from torchvision import transforms, datasets +from torchvision import datasets, transforms -from ...nm import DataLayerNM from .....core import * +from ...nm import DataLayerNM class ImageFolderDataLayer(DataLayerNM): diff --git a/nemo/nemo/backends/pytorch/tutorials/chatbot/modules.py b/nemo/nemo/backends/pytorch/tutorials/chatbot/modules.py index f4dd8828d5a9e..6a195342a99b1 100644 --- a/nemo/nemo/backends/pytorch/tutorials/chatbot/modules.py +++ b/nemo/nemo/backends/pytorch/tutorials/chatbot/modules.py @@ -2,16 +2,16 @@ https://pytorch.org/tutorials/beginner/chatbot_tutorial.html """ import random -from typing import Mapping, Iterable, Optional +from typing import Iterable, Mapping, Optional import torch as t import torch.nn as nn import torch.nn.functional as F -from ..chatbot import data -from ...nm import TrainableNM, DataLayerNM, LossNM from .....core import DeviceType from .....core.neural_types import * +from ...nm import DataLayerNM, LossNM, TrainableNM +from ..chatbot import data class DialogDataLayer(DataLayerNM): diff --git a/nemo/nemo/backends/pytorch/tutorials/toys.py b/nemo/nemo/backends/pytorch/tutorials/toys.py index 4813c0a956d11..a7e1b3ec8ca96 100644 --- a/nemo/nemo/backends/pytorch/tutorials/toys.py +++ b/nemo/nemo/backends/pytorch/tutorials/toys.py @@ -5,9 +5,9 @@ import torch.nn as nn import torch.utils.data as t_utils -from ..nm import TrainableNM, DataLayerNM, LossNM -from ....core import NeuralModule, DeviceType +from ....core import DeviceType, NeuralModule from ....core.neural_types import * +from ..nm import DataLayerNM, LossNM, TrainableNM class TaylorNet(TrainableNM): # Note inheritance from TrainableNM diff --git a/nemo/nemo/core/callbacks.py b/nemo/nemo/core/callbacks.py index 5b5503a1ad2b1..f62bce32976a4 100644 --- a/nemo/nemo/core/callbacks.py +++ b/nemo/nemo/core/callbacks.py @@ -1,11 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation -from abc import ABC, abstractmethod -from collections import namedtuple import glob import os import sys import time import warnings +from abc import ABC, abstractmethod +from collections import namedtuple import nemo diff --git a/nemo/nemo/core/neural_factory.py b/nemo/nemo/core/neural_factory.py index 9f0ca320e0f0c..cb80b2413c5d2 100644 --- a/nemo/nemo/core/neural_factory.py +++ b/nemo/nemo/core/neural_factory.py @@ -9,18 +9,19 @@ 'DeploymentFormat', ] -from abc import ABC, abstractmethod import random -from typing import List, Optional import warnings - +from abc import ABC, abstractmethod from enum import Enum +from typing import List, Optional + import numpy as np import nemo + +from ..utils import ExpManager from .callbacks import ActionCallback, EvaluatorCallback from .neural_types import * -from ..utils import ExpManager class DeploymentFormat(Enum): diff --git a/nemo/nemo/core/neural_modules.py b/nemo/nemo/core/neural_modules.py index 0ef39bebd155a..30b1044fa3a46 100644 --- a/nemo/nemo/core/neural_modules.py +++ b/nemo/nemo/core/neural_modules.py @@ -3,23 +3,23 @@ __all__ = ['WeightShareTransform', 'NeuralModule'] +import collections +import uuid from abc import ABC, abstractmethod from collections import namedtuple from enum import Enum from inspect import getargvalues, stack -from typing import Optional, Dict, Set, Tuple, List -import uuid -import collections +from typing import Dict, List, Optional, Set, Tuple from nemo.core import NeuralModuleFactory -from .neural_factory import Optimization, DeviceType +from .neural_factory import DeviceType, Optimization from .neural_types import ( CanNotInferResultNeuralType, - NeuralType, - NeuralTypeComparisonResult, NeuralPortNameMismatchError, NeuralPortNmTensorMismatchError, + NeuralType, + NeuralTypeComparisonResult, NmTensor, ) diff --git a/nemo/nemo/core/neural_types.py b/nemo/nemo/core/neural_types.py index 8b7a3dd0aa42a..7c2b1a547f005 100644 --- a/nemo/nemo/core/neural_types.py +++ b/nemo/nemo/core/neural_types.py @@ -32,8 +32,8 @@ 'CanNotInferResultNeuralType', ] -from enum import Enum import uuid +from enum import Enum class BaseTag(object): diff --git a/nemo/nemo/utils/exp_logging.py b/nemo/nemo/utils/exp_logging.py index 95479cb82997a..32d1abeb963ac 100644 --- a/nemo/nemo/utils/exp_logging.py +++ b/nemo/nemo/utils/exp_logging.py @@ -1,11 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation import logging import os -import time -from shutil import copyfile import subprocess import sys +import time import warnings +from shutil import copyfile import nemo diff --git a/nemo/nemo/utils/lr_policies.py b/nemo/nemo/utils/lr_policies.py index f65d044083e50..934a867bf64ee 100644 --- a/nemo/nemo/utils/lr_policies.py +++ b/nemo/nemo/utils/lr_policies.py @@ -8,11 +8,10 @@ 'SquareRootAnnealing', ] -import math -from abc import ABC, abstractmethod - import inspect +import math import sys +from abc import ABC, abstractmethod class _LRPolicy(ABC): diff --git a/scripts/build_lm_text.py b/scripts/build_lm_text.py index b11a261a7edd1..02bfa6c1aa4f6 100644 --- a/scripts/build_lm_text.py +++ b/scripts/build_lm_text.py @@ -1,7 +1,7 @@ -import pandas as pd -import os import argparse +import os +import pandas as pd if __name__ == '__main__': parser = argparse.ArgumentParser( diff --git a/scripts/export_bert_to_trt.py b/scripts/export_bert_to_trt.py index 22dade746bc00..2301d79e25eb9 100644 --- a/scripts/export_bert_to_trt.py +++ b/scripts/export_bert_to_trt.py @@ -18,8 +18,9 @@ import json import re -import torch import numpy as np +import torch + import tensorrt as trt nvinfer = ctypes.CDLL("libnvinfer_plugin.so", mode=ctypes.RTLD_GLOBAL) diff --git a/scripts/export_jasper_onnx_to_trt.py b/scripts/export_jasper_onnx_to_trt.py index b30f13fe9a877..5b5c09760c76f 100644 --- a/scripts/export_jasper_onnx_to_trt.py +++ b/scripts/export_jasper_onnx_to_trt.py @@ -1,6 +1,7 @@ import argparse import onnx + import tensorrt as trt diff --git a/scripts/export_jasper_to_onnx.py b/scripts/export_jasper_to_onnx.py index eb9d410da8bfa..398daf7f78f07 100644 --- a/scripts/export_jasper_to_onnx.py +++ b/scripts/export_jasper_to_onnx.py @@ -1,11 +1,12 @@ # Copyright (c) 2019 NVIDIA Corporation import argparse -import nemo -import nemo_asr import torch from ruamel.yaml import YAML +import nemo +import nemo_asr + def get_parser(): parser = argparse.ArgumentParser( diff --git a/scripts/get_aishell_data.py b/scripts/get_aishell_data.py index c22e3a6eb5247..3ad21d939a40b 100644 --- a/scripts/get_aishell_data.py +++ b/scripts/get_aishell_data.py @@ -3,11 +3,11 @@ # USAGE: python get_aishell_data.py --data_root= import argparse +import json import os -import urllib.request -import tarfile import subprocess -import json +import tarfile +import urllib.request parser = argparse.ArgumentParser(description='Aishell Data download') parser.add_argument("--data_root", required=True, default=None, type=str) diff --git a/scripts/get_databaker_data.py b/scripts/get_databaker_data.py index 6c0f5f81fd131..2938042f13ef9 100644 --- a/scripts/get_databaker_data.py +++ b/scripts/get_databaker_data.py @@ -18,18 +18,17 @@ import argparse import glob +import json import os +import random import urllib.request - from concurrent.futures import ProcessPoolExecutor from functools import partial -from pypinyin import lazy_pinyin, Style + import librosa -import json -import random +from pypinyin import Style, lazy_pinyin from tqdm import tqdm - URLS = { 'DATABAKER_CSMSC': "https://weixinxcxdb.oss-cn-beijing.aliyuncs.com/gwYinPinKu/BZNSYP.rar" } diff --git a/scripts/get_decoder_params_from_bert.py b/scripts/get_decoder_params_from_bert.py index 34b70612d11ed..f4f35aacf5cb7 100644 --- a/scripts/get_decoder_params_from_bert.py +++ b/scripts/get_decoder_params_from_bert.py @@ -1,7 +1,8 @@ +import argparse + import torch from transformers import BERT_PRETRAINED_MODEL_ARCHIVE_MAP from transformers.file_utils import cached_path -import argparse state_dict_mappings = { 'gamma': 'weight', diff --git a/scripts/get_librispeech_data.py b/scripts/get_librispeech_data.py index 4d18891821e88..8dde95401bd9d 100755 --- a/scripts/get_librispeech_data.py +++ b/scripts/get_librispeech_data.py @@ -7,14 +7,15 @@ # You can also put more than one data_set comma-separated: # --data_set=dev_clean,train_clean_100 import argparse -import os -import urllib.request -import tarfile import fnmatch -import subprocess import json -from tqdm import tqdm +import os +import subprocess +import tarfile +import urllib.request + from sox import Transformer +from tqdm import tqdm parser = argparse.ArgumentParser(description='LibriSpeech Data download') parser.add_argument("--data_root", required=True, default=None, type=str) diff --git a/scripts/get_ljspeech_data.py b/scripts/get_ljspeech_data.py index a6a94766c9984..7ca160921a95d 100644 --- a/scripts/get_ljspeech_data.py +++ b/scripts/get_ljspeech_data.py @@ -8,6 +8,7 @@ import random import tarfile import urllib.request + from scipy.io.wavfile import read URL = "https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2" diff --git a/scripts/get_tatoeba_data.py b/scripts/get_tatoeba_data.py index e287b047fb391..ad549d7cb662f 100644 --- a/scripts/get_tatoeba_data.py +++ b/scripts/get_tatoeba_data.py @@ -18,7 +18,6 @@ import re import string import urllib.request - from collections import Counter URL = {'tatoeba': 'https://downloads.tatoeba.org/exports/sentences.csv'} diff --git a/scripts/get_timit_data.py b/scripts/get_timit_data.py index 95d5db59a12c1..285ea68ed04d9 100644 --- a/scripts/get_timit_data.py +++ b/scripts/get_timit_data.py @@ -3,15 +3,16 @@ # --data_new_root= import argparse -import os -import urllib.request -import tarfile import fnmatch -import subprocess import json -from tqdm import tqdm -from sox import Transformer +import os +import subprocess import sys +import tarfile +import urllib.request + +from sox import Transformer +from tqdm import tqdm # # This script proposes to create the *json manifest necessary to use the TIMIT diff --git a/scripts/process_aishell2_data.py b/scripts/process_aishell2_data.py index 4f2456190cf0c..a3a4a6f717e5a 100644 --- a/scripts/process_aishell2_data.py +++ b/scripts/process_aishell2_data.py @@ -4,10 +4,10 @@ # --audio_folder= # --dest_folder= import argparse -import os -import sys import json +import os import subprocess +import sys parser = argparse.ArgumentParser(description="Processing Aishell2 Data") parser.add_argument( diff --git a/scripts/process_an4_data.py b/scripts/process_an4_data.py index 7bf81abb3925a..c802433fdf9aa 100644 --- a/scripts/process_an4_data.py +++ b/scripts/process_an4_data.py @@ -14,10 +14,11 @@ import argparse import glob import json -import librosa import os import subprocess import tarfile + +import librosa import wget parser = argparse.ArgumentParser( diff --git a/scripts/process_fisher_data.py b/scripts/process_fisher_data.py index be9976d385e35..defb8ac0bb733 100644 --- a/scripts/process_fisher_data.py +++ b/scripts/process_fisher_data.py @@ -20,8 +20,8 @@ import json import os import re +from math import ceil, floor -from math import floor, ceil import numpy as np import scipy.io.wavfile as wavfile from tqdm import tqdm diff --git a/scripts/process_hub5_data.py b/scripts/process_hub5_data.py index 9a4c87fb79c15..6186078f7873a 100644 --- a/scripts/process_hub5_data.py +++ b/scripts/process_hub5_data.py @@ -19,11 +19,11 @@ import re import subprocess import sys - from collections import namedtuple -from math import floor, ceil -import numpy as np +from math import ceil, floor from operator import attrgetter + +import numpy as np import scipy.io.wavfile as wavfile from tqdm import tqdm diff --git a/tests/common_setup.py b/tests/common_setup.py index 69ecbc1b2718a..95f91f81d8d16 100644 --- a/tests/common_setup.py +++ b/tests/common_setup.py @@ -1,11 +1,12 @@ # Copyright (c) 2019 NVIDIA Corporation -import unittest import os -from .context import nemo +import unittest # from nemo.backends.pytorch.nm import TrainableNM import torch.nn as nn +from .context import nemo + class NeMoUnitTest(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/context.py b/tests/context.py index b23f84516673a..9bdb8f9e058a0 100644 --- a/tests/context.py +++ b/tests/context.py @@ -1,10 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation +import os +import sys + import nemo import nemo_asr import nemo_nlp import nemo_tts -import os -import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) diff --git a/tests/other/jasper.py b/tests/other/jasper.py index 336e497faab9e..b122c446a7123 100644 --- a/tests/other/jasper.py +++ b/tests/other/jasper.py @@ -1,10 +1,4 @@ # Copyright (c) 2019 NVIDIA Corporation -from nemo.backends.pytorch.asr.helpers import ( - monitor_asr_train_progress, - process_evaluation_batch, - process_evaluation_epoch, -) -import nemo import argparse import os import sys @@ -12,6 +6,13 @@ import toml from tensorboardX import SummaryWriter +import nemo +from nemo.backends.pytorch.asr.helpers import ( + monitor_asr_train_progress, + process_evaluation_batch, + process_evaluation_epoch, +) + sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) ) diff --git a/tests/other/jasper_zero_dl.py b/tests/other/jasper_zero_dl.py index b114ed1ab1e9b..ef4f9d5db4855 100644 --- a/tests/other/jasper_zero_dl.py +++ b/tests/other/jasper_zero_dl.py @@ -1,7 +1,4 @@ # Copyright (c) 2019 NVIDIA Corporation -from nemo.backends.pytorch.asr.helpers import monitor_asr_train_progress -from nemo.core.neural_types import * -import nemo import argparse import os import sys @@ -10,6 +7,10 @@ import torch from tensorboardX import SummaryWriter +import nemo +from nemo.backends.pytorch.asr.helpers import monitor_asr_train_progress +from nemo.core.neural_types import * + sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')) ) diff --git a/tests/test_actions_api.py b/tests/test_actions_api.py index 0d0f178b1533c..30a8cb7739ac1 100644 --- a/tests/test_actions_api.py +++ b/tests/test_actions_api.py @@ -1,9 +1,9 @@ # Copyright (c) 2019 NVIDIA Corporation -import unittest import os +import unittest -from .context import nemo from .common_setup import NeMoUnitTest +from .context import nemo class TestTrainers(NeMoUnitTest): diff --git a/tests/test_asr.py b/tests/test_asr.py index dfeefc0b9edbd..124e0ae69986b 100644 --- a/tests/test_asr.py +++ b/tests/test_asr.py @@ -4,12 +4,16 @@ import tarfile import unittest -from nemo.core import DeviceType -from nemo_asr.parts import AudioDataset, WaveformFeaturizer -from nemo_asr.parts import collections -from nemo_asr.parts import parsers from ruamel.yaml import YAML +from nemo.core import DeviceType +from nemo_asr.parts import ( + AudioDataset, + WaveformFeaturizer, + collections, + parsers, +) + from .common_setup import NeMoUnitTest from .context import nemo, nemo_asr diff --git a/tests/test_bert.py b/tests/test_bert.py index b5a4ade0dce0f..9d3f108f6de90 100644 --- a/tests/test_bert.py +++ b/tests/test_bert.py @@ -1,9 +1,9 @@ # Copyright (c) 2019 NVIDIA Corporation -import unittest import os +import unittest -from .context import nemo, nemo_nlp from .common_setup import NeMoUnitTest +from .context import nemo, nemo_nlp class TestBert(NeMoUnitTest): diff --git a/tests/test_infer.py b/tests/test_infer.py index 789ba13268820..d820c40bdb5e0 100644 --- a/tests/test_infer.py +++ b/tests/test_infer.py @@ -4,8 +4,8 @@ from nemo.backends.pytorch.nm import NonTrainableNM from nemo.core.neural_types import * -from .context import nemo from .common_setup import NeMoUnitTest +from .context import nemo class AddsTen(NonTrainableNM): diff --git a/tests/test_neural_factory.py b/tests/test_neural_factory.py index 95b0cb5e1c5c7..8463456c91615 100644 --- a/tests/test_neural_factory.py +++ b/tests/test_neural_factory.py @@ -1,7 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation import unittest -from .context import nemo + from .common_setup import NeMoUnitTest +from .context import nemo class TestNeuralFactory(NeMoUnitTest): diff --git a/tests/test_neural_modules.py b/tests/test_neural_modules.py index 89e459a912c13..a294b095e4947 100644 --- a/tests/test_neural_modules.py +++ b/tests/test_neural_modules.py @@ -1,8 +1,10 @@ # Copyright (c) 2019 NVIDIA Corporation import unittest + from nemo.backends.pytorch.nm import TrainableNM -from .context import nemo + from .common_setup import NeMoUnitTest +from .context import nemo class TestNM1(TrainableNM): diff --git a/tests/test_neural_types.py b/tests/test_neural_types.py index 59465b01012e8..0b23e93246d5a 100644 --- a/tests/test_neural_types.py +++ b/tests/test_neural_types.py @@ -5,10 +5,11 @@ from ruamel.yaml import YAML -from .context import nemo, nemo_asr -from nemo.core import * import nemo_asr +from nemo.core import * + from .common_setup import NeMoUnitTest +from .context import nemo, nemo_asr class TestNeuralTypes(NeMoUnitTest): diff --git a/tests/test_policies.py b/tests/test_policies.py index 73e55bd9727a3..27447c345e9f3 100644 --- a/tests/test_policies.py +++ b/tests/test_policies.py @@ -1,8 +1,8 @@ import unittest from nemo.utils.lr_policies import ( - SquareAnnealing, CosineAnnealing, + SquareAnnealing, WarmupAnnealing, ) diff --git a/tests/test_pytorch_trainers.py b/tests/test_pytorch_trainers.py index 8c040ffaa8d70..684e9b90faa3c 100644 --- a/tests/test_pytorch_trainers.py +++ b/tests/test_pytorch_trainers.py @@ -1,7 +1,8 @@ # Copyright (c) 2019 NVIDIA Corporation import unittest -from .context import nemo + from .common_setup import NeMoUnitTest +from .context import nemo class TestPytorchTrainers(NeMoUnitTest): diff --git a/tests/test_spc_tokenizer.py b/tests/test_spc_tokenizer.py index 8cf7268ef86b9..23a99f2398e18 100644 --- a/tests/test_spc_tokenizer.py +++ b/tests/test_spc_tokenizer.py @@ -1,10 +1,11 @@ # Copyright (c) 2019 NVIDIA Corporation import unittest -from .context import nemo -from .common_setup import NeMoUnitTest from nemo_nlp import SentencePieceTokenizer +from .common_setup import NeMoUnitTest +from .context import nemo + class TestSPCTokenizer(NeMoUnitTest): def test_add_special_tokens(self): diff --git a/tests/test_squad.py b/tests/test_squad.py index fb636d8fc17d2..f116319ab4008 100644 --- a/tests/test_squad.py +++ b/tests/test_squad.py @@ -4,14 +4,15 @@ import shutil import sys -from .context import nemo, nemo_nlp -from .common_setup import NeMoUnitTest from nemo.utils.lr_policies import get_lr_policy from nemo_nlp.utils.callbacks.squad import ( - eval_iter_callback, eval_epochs_done_callback, + eval_iter_callback, ) +from .common_setup import NeMoUnitTest +from .context import nemo, nemo_nlp + class TestSquad(NeMoUnitTest): @classmethod diff --git a/tests/test_tts.py b/tests/test_tts.py index 2bde88b3525d8..d94d985f6d11d 100644 --- a/tests/test_tts.py +++ b/tests/test_tts.py @@ -2,8 +2,8 @@ import os import tarfile -from .context import nemo, nemo_asr, nemo_tts from .common_setup import NeMoUnitTest +from .context import nemo, nemo_asr, nemo_tts class TestTTSPytorch(NeMoUnitTest): diff --git a/tests/test_tutorials_pytorch.py b/tests/test_tutorials_pytorch.py index b457201d453c4..13853029ca95d 100644 --- a/tests/test_tutorials_pytorch.py +++ b/tests/test_tutorials_pytorch.py @@ -1,8 +1,10 @@ # Copyright (c) 2019 NVIDIA Corporation import unittest -from .context import nemo + from nemo.backends.pytorch.tutorials.chatbot.data import loadPrepareData + from .common_setup import NeMoUnitTest +from .context import nemo class TestPytorchChatBotTutorial(NeMoUnitTest): diff --git a/tests/test_weight_share.py b/tests/test_weight_share.py index e5da1651e1afb..142ab47f4dcc5 100644 --- a/tests/test_weight_share.py +++ b/tests/test_weight_share.py @@ -1,15 +1,16 @@ # Copyright (c) 2019 NVIDIA Corporation -from typing import Dict import unittest +from typing import Dict import numpy as np -from ruamel.yaml import YAML import torch +from ruamel.yaml import YAML -from .context import nemo, nemo_asr from nemo.core import WeightShareTransform from nemo.core.neural_types import * + from .common_setup import NeMoUnitTest +from .context import nemo, nemo_asr class TestWeightSharing(NeMoUnitTest): diff --git a/tests/test_zeroDS.py b/tests/test_zeroDS.py index 0ca1cbed93081..3640213921aa6 100644 --- a/tests/test_zeroDS.py +++ b/tests/test_zeroDS.py @@ -1,12 +1,14 @@ -import unittest import os import tarfile +import unittest + import torch from ruamel.yaml import YAML from nemo.core.neural_types import * -from .context import nemo, nemo_asr + from .common_setup import NeMoUnitTest +from .context import nemo, nemo_asr class TestZeroDL(NeMoUnitTest):