diff --git a/Dockerfile b/Dockerfile
index 08d8cfefb989..8dc275049631 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:21.02-py3
+ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:21.03-py3
 
 
 # build an image that includes only the nemo dependencies, ensures that dependencies
@@ -32,6 +32,9 @@ RUN apt-get update && \
     python-dev ffmpeg && \
     rm -rf /var/lib/apt/lists/*
 
+# uninstall stuff from base container
+RUN pip uninstall -y sacrebleu torchtext
+
 # build torchaudio (change latest release version to match pytorch)
 WORKDIR /tmp/torchaudio_build
 RUN git clone --depth 1 --branch release/0.7 https://github.com/pytorch/audio.git && \
@@ -39,6 +42,15 @@ RUN git clone --depth 1 --branch release/0.7 https://github.com/pytorch/audio.gi
     BUILD_SOX=1 python setup.py install && \
     cd .. && rm -r audio
 
+# TODO: remove when 21.04 container is released
+# build torchtext
+WORKDIR /tmp/torchtext_build
+RUN git clone --branch v0.8.1 https://github.com/pytorch/text.git && \
+    cd text && \
+    git submodule update --init --recursive && \
+    python setup.py clean install && \
+    cd .. && rm -r text 
+
 # build RNN-T loss
 WORKDIR /workspace/deps/rnnt
 RUN COMMIT_SHA=f546575109111c455354861a0567c8aa794208a2 && \
@@ -67,9 +79,6 @@ RUN COMMIT_SHA=f546575109111c455354861a0567c8aa794208a2 && \
     python3 setup.py install && \
     rm -rf ../tests test ../tensorflow_binding
 
-# uninstall stuff from base container
-RUN pip uninstall -y sacrebleu
-
 # install nemo dependencies
 WORKDIR /tmp/nemo
 COPY requirements .
@@ -101,7 +110,7 @@ RUN --mount=from=nemo-src,target=/tmp/nemo cd /tmp/nemo && pip install ".[all]"
     python -c "import nemo.collections.nlp as nemo_nlp" && \
     python -c "import nemo.collections.tts as nemo_tts"
 
-# TODO: Remove once 20.04 container is base container
+# TODO: Remove once 21.04 container is base container
 # install latest numba version
 RUN conda update -c numba numba -y
 
diff --git a/Jenkinsfile b/Jenkinsfile
index d793fe0a2416..f7f3bd855d9c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,7 +1,7 @@
 pipeline {
   agent {
         docker {
-      image 'nvcr.io/nvidia/pytorch:21.02-py3'
+      image 'nvcr.io/nvidia/pytorch:21.03-py3'
       args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home/TestData:/home/TestData -v $HOME/.cache/torch:/root/.cache/torch --shm-size=8g'
         }
   }
diff --git a/tests/collections/nlp/test_nlp_exportables.py b/tests/collections/nlp/test_nlp_exportables.py
index 139357e8a8c3..a9eb4a83c9ee 100644
--- a/tests/collections/nlp/test_nlp_exportables.py
+++ b/tests/collections/nlp/test_nlp_exportables.py
@@ -101,6 +101,7 @@ def test_IntentSlotClassificationModel_export_to_onnx(self, dummy_data):
 
     @pytest.mark.run_only_on('GPU')
     @pytest.mark.unit
+    @pytest.mark.skip('Are we exporting to torchscript?')
     def test_IntentSlotClassificationModel_export_to_torchscript(self, dummy_data):
         with tempfile.TemporaryDirectory() as tmpdir:
             wget.download(