diff --git a/examples/nlp/language_modeling/megatron_gpt_pretraining.py b/examples/nlp/language_modeling/megatron_gpt_pretraining.py index 1ac12b422108..596e3c59e753 100644 --- a/examples/nlp/language_modeling/megatron_gpt_pretraining.py +++ b/examples/nlp/language_modeling/megatron_gpt_pretraining.py @@ -13,6 +13,7 @@ # limitations under the License. +import torch.multiprocessing as mp from omegaconf.omegaconf import OmegaConf, open_dict from pytorch_lightning import Trainer from pytorch_lightning.plugins.environments import TorchElasticEnvironment @@ -29,6 +30,8 @@ from nemo.utils import logging from nemo.utils.exp_manager import exp_manager +mp.set_start_method("spawn", force=True) + @hydra_runner(config_path="conf", config_name="megatron_gpt_config") def main(cfg) -> None: diff --git a/nemo/collections/nlp/data/language_modeling/megatron/indexed_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/indexed_dataset.py index f45d9b7155a8..b2c5f4976f32 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/indexed_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/indexed_dataset.py @@ -500,10 +500,10 @@ def __init__(self, path, skip_warmup=False): def __getstate__(self): return self._path - # def __setstate__(self, state): - # self._do_init(state) + def __setstate__(self, state): + self._do_init(state) - def _do_init(self, path, skip_warmup): + def _do_init(self, path, skip_warmup=True): self._path = path self._index = self.Index(index_file_path(self._path), skip_warmup) diff --git a/nemo/package_info.py b/nemo/package_info.py index 56e296062fa7..561b82b3c37f 100644 --- a/nemo/package_info.py +++ b/nemo/package_info.py @@ -14,7 +14,7 @@ MAJOR = 1 -MINOR = 16 +MINOR = 17 PATCH = 0 PRE_RELEASE = 'rc0' diff --git a/tutorials/AudioTranslationSample.ipynb b/tutorials/AudioTranslationSample.ipynb index 12b354725982..c44eefd8aeca 100644 --- a/tutorials/AudioTranslationSample.ipynb +++ b/tutorials/AudioTranslationSample.ipynb @@ -280,4 +280,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/tutorials/VoiceSwapSample.ipynb b/tutorials/VoiceSwapSample.ipynb index 4f22e2cb82a6..bd53a2455b71 100644 --- a/tutorials/VoiceSwapSample.ipynb +++ b/tutorials/VoiceSwapSample.ipynb @@ -325,4 +325,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/tutorials/tts/FastPitch_ChineseTTS_Training.ipynb b/tutorials/tts/FastPitch_ChineseTTS_Training.ipynb index 442b74742a37..e11a4a50dc17 100644 --- a/tutorials/tts/FastPitch_ChineseTTS_Training.ipynb +++ b/tutorials/tts/FastPitch_ChineseTTS_Training.ipynb @@ -61,7 +61,7 @@ "# !pip install wget text-unidecode matplotlib>=3.3.2\n", "\n", "## Install NeMo\n", - "BRANCH = 'r1.16.0'\n", + "BRANCH = 'main'\n", "# !python -m pip install \"git+https://github.com/NVIDIA/NeMo.git@${BRANCH}#egg=nemo_toolkit[all]\"\n", "\n", "## Install pynini\n", diff --git a/tutorials/tts/FastPitch_Finetuning.ipynb b/tutorials/tts/FastPitch_Finetuning.ipynb index 7dc9bdbaf4a1..c41947624d16 100755 --- a/tutorials/tts/FastPitch_Finetuning.ipynb +++ b/tutorials/tts/FastPitch_Finetuning.ipynb @@ -110,9 +110,9 @@ "\n", "```\n", "{\"audio_filepath\": \"audio/dartagnan03part1_027_dumas_0047.wav\", \"text\": \"yes monsieur\", \"duration\": 1.04, \"text_no_preprocessing\": \"Yes, monsieur.\", \"text_normalized\": \"Yes, monsieur.\"}\n", - "```" - ] - }, + "```" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/tutorials/tts/FastPitch_GermanTTS_Training.ipynb b/tutorials/tts/FastPitch_GermanTTS_Training.ipynb index f22d8e320cba..fffa9e9f7aca 100644 --- a/tutorials/tts/FastPitch_GermanTTS_Training.ipynb +++ b/tutorials/tts/FastPitch_GermanTTS_Training.ipynb @@ -61,7 +61,7 @@ "# !pip install wget text-unidecode matplotlib>=3.3.2\n", "\n", "## Install NeMo\n", - "BRANCH = 'r1.16.0'\n", + "BRANCH = 'main'\n", "# !python -m pip install \"git+https://github.com/NVIDIA/NeMo.git@${BRANCH}#egg=nemo_toolkit[all]\"\n", "\n", "## Install pynini\n", diff --git a/tutorials/tts/NeMo_TTS_Primer.ipynb b/tutorials/tts/NeMo_TTS_Primer.ipynb index 12e3984fbe03..8100b3605364 100644 --- a/tutorials/tts/NeMo_TTS_Primer.ipynb +++ b/tutorials/tts/NeMo_TTS_Primer.ipynb @@ -129,7 +129,7 @@ "While this is the most common structure, there may be fewer or additional steps depending on the use case. For example, some languages do not require G2P and can instead rely on the model to convert raw text/graphemes to spectrogram.\n", "\n", "
\n", - "\n", + "\n", "
" ] }, @@ -199,9 +199,9 @@ " \n", "\n", "\n", - "The above examples may be slightly different than the output of the NeMo text normalization code. More details on NeMo text normalization can be found in the [TN documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/nlp/text_normalization/intro.html).\n", + "The above examples may be slightly different than the output of the NeMo text normalization code. More details on NeMo text normalization can be found in the [TN documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/nlp/text_normalization/intro.html).\n", "\n", - "A more comprehensive list of text normalization rules, examples, and languages are available in the [code](https://github.com/NVIDIA/NeMo/tree/main/nemo_text_processing/text_normalization).\n", + "A more comprehensive list of text normalization rules, examples, and languages are available in the [code](https://github.com/NVIDIA/NeMo-text-processing/tree/main/nemo_text_processing/text_normalization).\n", "\n" ] }, @@ -343,7 +343,7 @@ "\n", "For non-phonetic languages like English it is still possible to train a TTS model directly on the graphemes. But doing so will make the pronunciation of some words less accurate.\n", "\n", - "Details on how NeMo G2P works can be found in our [G2P documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/tts/g2p.html)." + "Details on how NeMo G2P works can be found in our [G2P documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/tts/g2p.html)." ] }, { @@ -756,7 +756,7 @@ "\n", "
\n", "
\n", - "\n", + "\n", "
https://wiki.hydrogenaud.io/index.php?title=File:Digital_wave.png\n", "
\n", "
" @@ -1050,7 +1050,7 @@ "\n", "
\n", "
\n", - "\n", + "\n", "
\n", "\n", "The model is fairly complex. At a high level, it contains:\n", @@ -1467,7 +1467,7 @@ "In NeMo we support [FastPitch](https://fastpitch.github.io/), a parallel transformer-based model with pitch and duration control and prediction.\n", "\n", "
\n", - "\n", + "\n", "
\n", "\n", "At a high level it contains:\n", @@ -1845,7 +1845,7 @@ "\n", "
\n", "
\n", - "\n", + "\n", "
Diagram of a dilated causal CNN
\n", "
\n", "
\n", @@ -1885,7 +1885,7 @@ "In addition to penalizing the model if the discriminator can classify the synthesized audio as fake, it also uses **feature matching loss** to penalize the model if the distribution of intermediate layer outputs in the discriminator networks differ between the real and synthesized audio.\n", "\n", "
\n", - "\n", + "\n", "
HiFi-Gan scale and period discriminators
\n", "
" ] @@ -1977,12 +1977,12 @@ } }, "source": [ - "To learn more about what TTS technology and models are available in NeMo, please look through our [documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/tts/intro.html#).\n", + "To learn more about what TTS technology and models are available in NeMo, please look through our [documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/tts/intro.html#).\n", "\n", - "To get more hands on experience with NeMo TTS, look through some of our other [tutorials](https://github.com/NVIDIA/NeMo/tree/main/tutorials/tts).\n", + "To get more hands on experience with NeMo TTS, look through some of our other [tutorials](https://github.com/NVIDIA/NeMo/tree/stable/tutorials/tts).\n", "\n", - "* Running pretrained models: [Inference_ModelSelect](https://github.com/NVIDIA/NeMo/blob/main/tutorials/tts/Inference_ModelSelect.ipynb)\n", - "* FastPitch [training](https://github.com/NVIDIA/NeMo/blob/main/tutorials/tts/FastPitch_MixerTTS_Training.ipynb) and [fine-tuning](https://github.com/NVIDIA/NeMo/blob/main/tutorials/tts/FastPitch_Finetuning.ipynb)\n", + "* Running pretrained models: [Inference_ModelSelect](https://github.com/NVIDIA/NeMo/blob/stable/tutorials/tts/Inference_ModelSelect.ipynb)\n", + "* FastPitch [training](https://github.com/NVIDIA/NeMo/blob/stable/tutorials/tts/FastPitch_MixerTTS_Training.ipynb) and [fine-tuning](https://github.com/NVIDIA/NeMo/blob/stable/tutorials/tts/FastPitch_Finetuning.ipynb)\n", "\n", "To learn how to deploy and serve your TTS models, visit [Riva](https://docs.nvidia.com/deeplearning/riva/index.html)." ]