From 7963b4ce2bb6f248f0a1c0a5420ba4690e45afce Mon Sep 17 00:00:00 2001 From: Boris Fomitchev Date: Mon, 31 Aug 2020 16:48:22 -0700 Subject: [PATCH 1/3] Propagating onnx.export() parameters; requiring torch>=1.6 --- nemo/core/classes/exportable.py | 26 +++++++++++++++----------- requirements/requirements.txt | 4 ++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/nemo/core/classes/exportable.py b/nemo/core/classes/exportable.py index 9424a889aaad..fe672875b98f 100644 --- a/nemo/core/classes/exportable.py +++ b/nemo/core/classes/exportable.py @@ -47,13 +47,17 @@ class Exportable(ABC): """ def export( - self, - output: str, - input_example=None, - output_example=None, - onnx_opset_version: int = 12, - try_script: bool = False, - set_eval: bool = True, + self, + output: str, + input_example=None, + output_example=None, + verbose=False, + export_params=True, + do_constant_folding=True, + keep_initializers_as_inputs=False, + onnx_opset_version: int = 12, + try_script: bool = False, + set_eval: bool = True, ): try: # Disable typechecks @@ -134,10 +138,10 @@ def export( output, input_names=input_names, output_names=output_names, - verbose=False, - export_params=True, - do_constant_folding=True, - keep_initializers_as_inputs=True, + verbose=verbose, + export_params=export_params, + do_constant_folding=do_constant_folding, + keep_initializers_as_inputs=keep_initializers_as_inputs, dynamic_axes=dynamic_axes, opset_version=onnx_opset_version, example_outputs=_out_example, diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c229303e7c88..e0a69d02aeed 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,11 +2,11 @@ numpy>=1.18.2 onnx>=1.7.0 pytorch-lightning==0.9.0 python-dateutil -torch +torch>=1.6.0 wget wrapt ruamel.yaml scikit-learn omegaconf==2.0.1rc12 hydra-core==1.0.0rc4 -transformers>=2.11.0 \ No newline at end of file +transformers>=2.11.0 From 3d9cf87955b2e2ca3b5b210fe16a5d214dee9d36 Mon Sep 17 00:00:00 2001 From: Boris Fomitchev Date: Tue, 1 Sep 2020 14:26:16 -0700 Subject: [PATCH 2/3] Adding all requirements install for Dockerfile and reinstall.sh, moving to 20.08 base --- Dockerfile | 7 +-- reinstall.sh | 18 +++++--- requirements/requirements.txt | 2 +- requirements/requirements_docker.txt | 66 ---------------------------- setup.py | 1 - 5 files changed, 17 insertions(+), 77 deletions(-) delete mode 100644 requirements/requirements_docker.txt diff --git a/Dockerfile b/Dockerfile index f7ab37b2a71e..2101f2944817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:20.07-py3 +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:20.08-py3 + # build an image that includes only the nemo dependencies, ensures that dependencies # are included first for optimal caching, and useful for building a development @@ -32,8 +33,8 @@ RUN apt-get update && \ # install nemo dependencies WORKDIR /tmp/nemo -COPY requirements/requirements_docker.txt requirements.txt -RUN pip install --disable-pip-version-check --no-cache-dir -r requirements.txt +COPY requirements . +RUN for f in requirements/*.txt; do pip install --disable-pip-version-check --no-cache-dir -r $f; done # copy nemo source into a scratch image FROM scratch as nemo-src diff --git a/reinstall.sh b/reinstall.sh index ff266836ecda..f5f9bbef09f8 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -1,16 +1,22 @@ #!/usr/bin/env bash set -e +PIP=pip + echo 'Uninstalling stuff' -pip uninstall -y nemo_toolkit +${PIP} uninstall -y nemo_toolkit # Kept for legacy purposes -pip uninstall -y nemo_asr -pip uninstall -y nemo_nlp -pip uninstall -y nemo_tts -pip uninstall -y nemo_simple_gan +${PIP} uninstall -y nemo_asr +${PIP} uninstall -y nemo_nlp +${PIP} uninstall -y nemo_tts +${PIP} uninstall -y nemo_simple_gan + +${PIP} install -U setuptools + +for f in requirements/*.txt; do ${PIP} install ${PIP_FLAGS}--disable-pip-version-check --no-cache-dir -r $f; done echo 'Installing stuff' -pip install -e ".[all]" +${PIP} install -e ".[all]" echo 'All done!' diff --git a/requirements/requirements.txt b/requirements/requirements.txt index e0a69d02aeed..6decd01bb2dd 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,7 +2,7 @@ numpy>=1.18.2 onnx>=1.7.0 pytorch-lightning==0.9.0 python-dateutil -torch>=1.6.0 +torch wget wrapt ruamel.yaml diff --git a/requirements/requirements_docker.txt b/requirements/requirements_docker.txt deleted file mode 100644 index c955f2af092e..000000000000 --- a/requirements/requirements_docker.txt +++ /dev/null @@ -1,66 +0,0 @@ -# core -numpy>=1.18.2 -onnx>=1.7.0 -pytorch-lightning>=0.8.5 -python-dateutil -torch -wget -wrapt -ruamel.yaml -scikit-learn -hydra-core==1.0.0rc2 - -# test -black -isort[requirements] < 5 -parameterized -pytest -pytest-runner -sphinx -sphinxcontrib-bibtex - - -# doc -latexcodec -sphinx_rtd_theme - - -# cv -pillow -torchvision - -# asr -braceexpand -editdistance -frozendict -inflect -kaldi-io -librosa -marshmallow -packaging -num2words -soundfile -sox -torch-stft -unidecode -webdataset -kaldi-python-io -scipy -pandas - -# nlp -boto3 -h5py -matplotlib -sentencepiece -torchtext -transformers>=2.11.0 -youtokentome -tqdm>=4.41.0 -rapidfuzz -gdown -megatron-lm>=1.1.3 - -# tts -pypinyin -attrdict diff --git a/setup.py b/setup.py index f2331942ec0e..49aea01bcd5f 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ def req_file(filename, folder="requirements"): extras_require = { # User packages - 'docker': req_file("requirements_docker.txt"), 'test': req_file("requirements_test.txt"), # Collections Packages 'asr': req_file("requirements_asr.txt"), From 1ed234c2cbbe463bb8254ec0883b84712f54f77c Mon Sep 17 00:00:00 2001 From: Boris Fomitchev Date: Tue, 1 Sep 2020 14:41:49 -0700 Subject: [PATCH 3/3] Bumping megatron-lm version Signed-off-by: Boris Fomitchev --- requirements/requirements_nlp.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements_nlp.txt b/requirements/requirements_nlp.txt index 60b5f92234af..18de20f9200f 100644 --- a/requirements/requirements_nlp.txt +++ b/requirements/requirements_nlp.txt @@ -9,5 +9,5 @@ numpy tqdm>=4.41.0 rapidfuzz gdown -megatron-lm>=1.1.3 +megatron-lm>=1.1.4 inflect