chore: 0.35.0 images (#9732)

* retag images to 0.35.0
determined-ai · Jul 25, 2024 · 40e1f56 · 40e1f56
1 parent f68e36e
commit 40e1f56
Show file tree

Hide file tree

Showing 35 changed files with 84 additions and 87 deletions.
diff --git a/.circleci/real_config.yml b/.circleci/real_config.yml
@@ -282,7 +282,7 @@ commands:
       - when:
           condition: <<parameters.tf2>>
           steps:
-            - run: docker pull determinedai/pytorch-ngc-dev:f20b027
+            - run: docker pull determinedai/pytorch-ngc:0.35.0
 
   login-docker:
     parameters:
@@ -2439,7 +2439,7 @@ jobs:
 
   test-unit-harness-gpu-tf:
     docker:
-      - image: determinedai/tensorflow-ngc-dev:f20b027
+      - image: determinedai/tensorflow-ngc:0.35.0
     resource_class: determined-ai/container-runner-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -2466,7 +2466,7 @@ jobs:
 
   test-unit-harness-pytorch2-gpu:
     docker:
-      - image: determinedai/pytorch-ngc-dev:f20b027
+      - image: determinedai/pytorch-ngc:0.35.0
     resource_class: determined-ai/container-runner-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -2493,7 +2493,7 @@ jobs:
 
   test-unit-harness-pytorch2-cpu:
     docker:
-      - image: determinedai/pytorch-ngc-dev:f20b027
+      - image: determinedai/pytorch-ngc:0.35.0
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
       - checkout
@@ -2519,7 +2519,7 @@ jobs:
 
   test-unit-harness-gpu-parallel:
     docker:
-      - image: determinedai/pytorch-ngc-dev:f20b027
+      - image: determinedai/pytorch-ngc:0.35.0
     resource_class: determined-ai/container-runner-multi-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -2546,7 +2546,7 @@ jobs:
 
   test-unit-harness-gpu-deepspeed:
     docker:
-      - image: determinedai/pytorch-ngc-dev:f20b027
+      - image: determinedai/pytorch-ngc:0.35.0
     resource_class: determined-ai/container-runner-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -3652,7 +3652,7 @@ jobs:
         type: string
         default: "1"
       environment-image:
-        default: determinedai/pytorch-ngc-dev:f20b027
+        default: determinedai/pytorch-ngc:0.35.0
         type: string
       accel-node-taints:
         type: string

diff --git a/.circleci/scripts/pull_image_daemonset.yaml b/.circleci/scripts/pull_image_daemonset.yaml
@@ -13,7 +13,7 @@ spec:
     spec:
       containers:
         - name: pull-docker-daemonset
-          image: determinedai/pytorch-ngc-dev:f20b027
+          image: determinedai/pytorch-ngc:0.35.0
           command: ["/bin/bash"]
           args: ["echo", "test"]
           resources:

diff --git a/docs/model-dev-guide/prepare-container/custom-env.rst b/docs/model-dev-guide/prepare-container/custom-env.rst
@@ -114,9 +114,9 @@ Default Images
    -  -  Environment
       -  File Name
    -  -  CPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  NVIDIA GPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  AMD GPUs
       -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4``
 
@@ -155,7 +155,7 @@ Example Dockerfile that installs custom ``conda``-, ``pip``-, and ``apt``-based
 .. code:: bash
 
    # Determined Image
-   FROM determinedai/tensorflow-ngc-dev:f20b027
+   FROM determinedai/tensorflow-ngc:0.35.0
 
    # Custom Configuration
    RUN apt-get update && \
@@ -216,7 +216,7 @@ environments using :ref:`custom images <custom-docker-images>`:
 .. code:: bash
 
    # Determined Image
-   FROM determinedai/pytorch-ngc-dev:f20b027
+   FROM determinedai/pytorch-ngc:0.35.0
 
    # Create a virtual environment
    RUN conda create -n myenv python=3.8

diff --git a/docs/model-dev-guide/prepare-container/tensorflow-support.rst b/docs/model-dev-guide/prepare-container/tensorflow-support.rst
@@ -20,7 +20,7 @@ Determined supports both TensorFlow 1 and 2. The version of TensorFlow used for
 experiment is controlled by the configured container image. Determined provides prebuilt Docker
 images that include TensorFlow 2+, 1.15, and 2.8, respectively:
 
--  ``determinedai/tensorflow-ngc-dev:f20b027``
+-  ``determinedai/tensorflow-ngc:0.35.0``
 -  ``determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-0.21.2``
 -  ``determinedai/environments:cuda-11.2-tf-2.8-gpu-0.29.1``
 

diff --git a/docs/reference/deploy/helm-config-reference.rst b/docs/reference/deploy/helm-config-reference.rst
@@ -197,13 +197,13 @@
 
    -  ``cpuImage``: Sets the default Docker image for all non-GPU tasks. If a Docker image is
       specified in the :ref:`experiment config <exp-environment-image>` this default is overriden.
-      Defaults to: ``determinedai/pytorch-ngc-dev:f20b027``.
+      Defaults to: ``determinedai/pytorch-ngc:0.35.0``.
 
    -  ``startupHook``: An optional inline script that will be executed as part of task set up.
 
    -  ``gpuImage``: Sets the default Docker image for all GPU tasks. If a Docker image is specified
       in the :ref:`experiment config <exp-environment-image>` this default is overriden. Defaults
-      to: ``determinedai/pytorch-ngc-dev:f20b027``.
+      to: ``determinedai/pytorch-ngc:0.35.0``.
 
    -  ``logPolicies``: Sets log policies for trials. For details, visit :ref:`log_policies
       <experiment-config-min-validation-period>`.

diff --git a/docs/reference/deploy/master-config-reference.rst b/docs/reference/deploy/master-config-reference.rst
@@ -89,12 +89,12 @@ configure different container images for NVIDIA GPU tasks using the ``cuda`` key
 Determined 0.17.6), CPU tasks using ``cpu`` key, and ROCm (AMD GPU) tasks using the ``rocm`` key.
 Default values:
 
--  ``determinedai/pytorch-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+-  ``determinedai/pytorch-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
 -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
 
 For TensorFlow users, we provide an image that must be referenced in the experiment configuration:
 
--  ``determinedai/tensorflow-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+-  ``determinedai/tensorflow-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
 
 ``environment_variables``
 =========================

diff --git a/docs/reference/experiment-config-reference.rst b/docs/reference/experiment-config-reference.rst
@@ -1353,12 +1353,12 @@ Optional. The Docker image to use when executing the workload. This image must b
 container images for NVIDIA GPU tasks using ``cuda`` key (``gpu`` prior to 0.17.6), CPU tasks using
 ``cpu`` key, and ROCm (AMD GPU) tasks using ``rocm`` key. Default values:
 
--  ``determinedai/pytorch-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+-  ``determinedai/pytorch-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
 -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
 
 For TensorFlow users, we provide an image that must be referenced in the experiment configuration:
 
--  ``determinedai/tensorflow-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+-  ``determinedai/tensorflow-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
 
 When the cluster is configured with :ref:`resource_manager.type: slurm
 <cluster-configuration-slurm>` and ``container_run_type: singularity``, images are executed using

diff --git a/docs/reference/job-config-reference.rst b/docs/reference/job-config-reference.rst
@@ -45,13 +45,13 @@ The following configuration settings are supported:
       different container images for NVIDIA GPU tasks using ``cuda`` key (``gpu`` prior to 0.17.6),
       CPU tasks using ``cpu`` key, and ROCm (AMD GPU) tasks using ``rocm`` key. Default values:
 
-      -  ``determinedai/pytorch-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+      -  ``determinedai/pytorch-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
       -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
 
       For TensorFlow users, we provide an image that must be referenced in the experiment
       configuration:
 
-      -  ``determinedai/tensorflow-ngc-dev:f20b027`` for NVIDIA GPUs and for CPUs.
+      -  ``determinedai/tensorflow-ngc:0.35.0`` for NVIDIA GPUs and for CPUs.
 
    -  ``force_pull_image``: Forcibly pull the image from the Docker registry and bypass the Docker
       cache. Defaults to ``false``.

diff --git a/docs/setup-cluster/deploy-cluster/slurm/singularity.rst b/docs/setup-cluster/deploy-cluster/slurm/singularity.rst
@@ -30,9 +30,9 @@ by default in this version of Determined are described below.
    -  -  Environment
       -  File Name
    -  -  CPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  NVIDIA GPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  AMD GPUs
       -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512``
 

diff --git a/docs/setup-cluster/gcp/install-gcp.rst b/docs/setup-cluster/gcp/install-gcp.rst
@@ -406,5 +406,5 @@ This command line will spin up a cluster of up to 2 A100s in the ``us-central1-c
       --compute-agent-instance-type a2-highgpu-1g --gpu-num 1 \
       --gpu-type nvidia-tesla-a100 \
       --region us-central1 --zone us-central1-c \
-      --gpu-env-image determinedai/pytorch-ngc-dev:f20b027 \
-      --cpu-env-image determinedai/pytorch-ngc-dev:f20b027
+      --gpu-env-image determinedai/pytorch-ngc:0.35.0 \
+      --cpu-env-image determinedai/pytorch-ngc:0.35.0
diff --git a/docs/setup-cluster/slurm/singularity.rst b/docs/setup-cluster/slurm/singularity.rst
@@ -30,9 +30,9 @@ by default in this version of Determined are described below.
    -  -  Environment
       -  File Name
    -  -  CPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  NVIDIA GPUs
-      -  ``determinedai/pytorch-ngc-dev:f20b027``
+      -  ``determinedai/pytorch-ngc:0.35.0``
    -  -  AMD GPUs
       -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512``
 

diff --git a/docs/setup-cluster/slurm/slurm-requirements.rst b/docs/setup-cluster/slurm/slurm-requirements.rst
@@ -510,7 +510,7 @@ platform. There may be additional per-user configuration that is required.
 
    .. code:: bash
 
-      image=determinedai/pytorch-ngc-dev:f20b027
+      image=determinedai/pytorch-ngc:0.35.0
       cd /shared/enroot/images
       enroot import docker://$image
       enroot create /shared/enroot/images/${image//[\/:]/\+}.sqsh

diff --git a/e2e_tests/tests/config.py b/e2e_tests/tests/config.py
@@ -14,12 +14,12 @@
 MAX_TRIAL_BUILD_SECS = 90
 
 
-DEFAULT_TF2_CPU_IMAGE = "determinedai/tensorflow-ngc-dev:f20b027"
-DEFAULT_TF2_GPU_IMAGE = "determinedai/tensorflow-ngc-dev:f20b027"
-DEFAULT_PT_CPU_IMAGE = "determinedai/pytorch-tensorflow-cpu-dev:f20b027"
-DEFAULT_PT_GPU_IMAGE = "determinedai/pytorch-tensorflow-cuda-dev:f20b027"
-DEFAULT_PT2_CPU_IMAGE = "determinedai/pytorch-ngc-dev:f20b027"
-DEFAULT_PT2_GPU_IMAGE = "determinedai/pytorch-ngc-dev:f20b027"
+DEFAULT_TF2_CPU_IMAGE = "determinedai/tensorflow-ngc:0.35.0"
+DEFAULT_TF2_GPU_IMAGE = "determinedai/tensorflow-ngc:0.35.0"
+DEFAULT_PT_CPU_IMAGE = "determinedai/pytorch-tensorflow-cpu:0.35.0"
+DEFAULT_PT_GPU_IMAGE = "determinedai/pytorch-tensorflow-cuda:0.35.0"
+DEFAULT_PT2_CPU_IMAGE = "determinedai/pytorch-ngc:0.35.0"
+DEFAULT_PT2_GPU_IMAGE = "determinedai/pytorch-ngc:0.35.0"
 
 TF2_CPU_IMAGE = os.environ.get("TF2_CPU_IMAGE") or DEFAULT_TF2_CPU_IMAGE
 TF2_GPU_IMAGE = os.environ.get("TF2_GPU_IMAGE") or DEFAULT_TF2_GPU_IMAGE

diff --git a/e2e_tests/tests/fixtures/ports-proxy/config.yaml b/e2e_tests/tests/fixtures/ports-proxy/config.yaml
@@ -23,7 +23,7 @@ max_restarts: 0
 
 # Hardcode the image because the new image has a bug. TODO fix this when the image bug is fixed.
 environment:
-  image: determinedai/pytorch-tensorflow-cpu-dev:f20b027
+  image: determinedai/pytorch-tensorflow-cpu:0.35.0
   proxy_ports:
     - proxy_port: 8000
       proxy_tcp: false

diff --git a/examples/computer_vision/iris_tf_keras/adaptive.yaml b/examples/computer_vision/iris_tf_keras/adaptive.yaml
@@ -4,8 +4,8 @@ data:
   test_url: http://download.tensorflow.org/data/iris_test.csv
 environment:
   image:
-    cpu: determinedai/tensorflow-ngc-dev:f20b027
-    gpu: determinedai/tensorflow-ngc-dev:f20b027
+    cpu: determinedai/tensorflow-ngc:0.35.0
+    gpu: determinedai/tensorflow-ngc:0.35.0
 hyperparameters:
   learning_rate:
     type: log

diff --git a/examples/computer_vision/iris_tf_keras/const.yaml b/examples/computer_vision/iris_tf_keras/const.yaml
@@ -4,8 +4,8 @@ data:
   test_url: http://download.tensorflow.org/data/iris_test.csv
 environment:
   image:
-    cpu: determinedai/tensorflow-ngc-dev:f20b027
-    gpu: determinedai/tensorflow-ngc-dev:f20b027
+    cpu: determinedai/tensorflow-ngc:0.35.0
+    gpu: determinedai/tensorflow-ngc:0.35.0
 hyperparameters:
   learning_rate: 1.0e-4
   learning_rate_decay: 1.0e-6

diff --git a/examples/computer_vision/iris_tf_keras/distributed.yaml b/examples/computer_vision/iris_tf_keras/distributed.yaml
@@ -4,8 +4,8 @@ data:
   test_url: http://download.tensorflow.org/data/iris_test.csv
 environment:
   image:
-    cpu: determinedai/tensorflow-ngc-dev:f20b027
-    gpu: determinedai/tensorflow-ngc-dev:f20b027
+    cpu: determinedai/tensorflow-ngc:0.35.0
+    gpu: determinedai/tensorflow-ngc:0.35.0
 hyperparameters:
   learning_rate: 1.0e-4
   learning_rate_decay: 1.0e-6

diff --git a/examples/computer_vision/iris_tf_keras/iris_tf_keras_cancelable.yaml b/examples/computer_vision/iris_tf_keras/iris_tf_keras_cancelable.yaml
@@ -4,8 +4,8 @@ data:
   test_url: http://download.tensorflow.org/data/iris_test.csv
 environment:
   image:
-    cpu: determinedai/tensorflow-ngc-dev:f20b027
-    gpu: determinedai/tensorflow-ngc-dev:f20b027
+    cpu: determinedai/tensorflow-ngc:0.35.0
+    gpu: determinedai/tensorflow-ngc:0.35.0
 resources:
   slots_per_trial: 8
   resource_pool: defq_GPU_cancelable

diff --git a/examples/computer_vision/iris_tf_keras/iris_tf_keras_high_priority.yaml b/examples/computer_vision/iris_tf_keras/iris_tf_keras_high_priority.yaml
@@ -4,8 +4,8 @@ data:
   test_url: http://download.tensorflow.org/data/iris_test.csv
 environment:
   image:
-    cpu: determinedai/tensorflow-ngc-dev:f20b027
-    gpu: determinedai/tensorflow-ngc-dev:f20b027
+    cpu: determinedai/tensorflow-ngc:0.35.0
+    gpu: determinedai/tensorflow-ngc:0.35.0
 resources:
   slots_per_trial: 8
   resource_pool: defq_GPU_hipri

diff --git a/examples/deepspeed_autotune/torchvision/core_api/deepspeed.yaml b/examples/deepspeed_autotune/torchvision/core_api/deepspeed.yaml
@@ -2,7 +2,7 @@ name: torchvision dsat core_api
 max_restarts: 0
 environment:
   image:
-    gpu: determinedai/pytorch-ngc-dev:f20b027
+    gpu: determinedai/pytorch-ngc:0.35.0
 resources:
   slots_per_trial: 2
   shm_size: 4294967296 # 4 GiB.

diff --git a/examples/deepspeed_autotune/torchvision/deepspeed_trial/deepspeed.yaml b/examples/deepspeed_autotune/torchvision/deepspeed_trial/deepspeed.yaml
@@ -2,7 +2,7 @@ name: torchvision dsat deepspeed_trial
 max_restarts: 0
 environment:
   image:
-    gpu: determinedai/pytorch-ngc-dev:f20b027
+    gpu: determinedai/pytorch-ngc:0.35.0
 resources:
   slots_per_trial: 2
   shm_size: 4294967296 # 4 GiB.

diff --git a/examples/hf_trainer_api/hf_image_classification/deepspeed.yaml b/examples/hf_trainer_api/hf_image_classification/deepspeed.yaml
@@ -6,7 +6,7 @@ environment:
     # You may need to modify this to match your network configuration.
     - NCCL_SOCKET_IFNAME=ens,eth,ib
   image:
-    gpu: determinedai/pytorch-ngc-dev:f20b027
+    gpu: determinedai/pytorch-ngc:0.35.0
 resources:
   slots_per_trial: 2
 searcher:

diff --git a/examples/hf_trainer_api/hf_language_modeling/deepspeed.yaml b/examples/hf_trainer_api/hf_language_modeling/deepspeed.yaml
@@ -6,7 +6,7 @@ environment:
     # You may need to modify this to match your network configuration.
     - NCCL_SOCKET_IFNAME=ens,eth,ib
   image:
-    gpu: determinedai/pytorch-ngc-dev:f20b027
+    gpu: determinedai/pytorch-ngc:0.35.0
 resources:
   slots_per_trial: 2
 searcher:

diff --git a/harness/tests/experiment/fixtures/ancient-checkpoints/0.17.6-keras/metadata.json b/harness/tests/experiment/fixtures/ancient-checkpoints/0.17.6-keras/metadata.json
@@ -39,8 +39,8 @@
       },
       "force_pull_image": false,
       "image": {
-        "cpu": "determinedai/tensorflow-ngc-dev:f20b027",
-        "cuda": "determinedai/tensorflow-ngc-dev:f20b027",
+        "cpu": "determinedai/tensorflow-ngc:0.35.0",
+        "cuda": "determinedai/tensorflow-ngc:0.35.0",
         "rocm": "determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512"
       },
       "pod_spec": null,

diff --git a/harness/tests/experiment/fixtures/ancient-checkpoints/0.17.6-pytorch/metadata.json b/harness/tests/experiment/fixtures/ancient-checkpoints/0.17.6-pytorch/metadata.json
@@ -38,8 +38,8 @@
       },
       "force_pull_image": false,
       "image": {
-        "cpu": "determinedai/tensorflow-ngc-dev:f20b027",
-        "cuda": "determinedai/tensorflow-ngc-dev:f20b027",
+        "cpu": "determinedai/tensorflow-ngc:0.35.0",
+        "cuda": "determinedai/tensorflow-ngc:0.35.0",
         "rocm": "determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512"
       },
       "pod_spec": null,

diff --git a/harness/tests/fixtures/checkpoint.json b/harness/tests/fixtures/checkpoint.json
@@ -69,8 +69,8 @@
           },
           "force_pull_image":false,
           "image":{
-            "cpu":"determinedai/pytorch-ngc-dev:f20b027",
-            "cuda":"determinedai/pytorch-ngc-dev:f20b027",
+            "cpu":"determinedai/pytorch-ngc:0.35.0",
+            "cuda":"determinedai/pytorch-ngc:0.35.0",
             "rocm":"determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512"
           },
           "pod_spec":null,

diff --git a/helm/charts/determined/values.yaml b/helm/charts/determined/values.yaml
@@ -31,8 +31,8 @@ defaultImages:
   kubeSchedulerPreemption: "determinedai/kube-scheduler:0.17.0"
 
   # default images for CPU and GPU environments
-  cpuImage: "determinedai/pytorch-ngc-dev:f20b027"
-  gpuImage: "determinedai/pytorch-ngc-dev:f20b027"
+  cpuImage: "determinedai/pytorch-ngc:0.35.0"
+  gpuImage: "determinedai/pytorch-ngc:0.35.0"
 
 # Install Determined enterprise edition.
 enterpriseEdition: false

diff --git a/master/pkg/schemas/expconf/const.go b/master/pkg/schemas/expconf/const.go
@@ -8,7 +8,7 @@ const (
 
 // Default task environment docker image names.
 const (
-	CPUImage  = "determinedai/pytorch-ngc-dev:f20b027"
-	CUDAImage = "determinedai/pytorch-ngc-dev:f20b027"
+	CPUImage  = "determinedai/pytorch-ngc:0.35.0"
+	CUDAImage = "determinedai/pytorch-ngc:0.35.0"
 	ROCMImage = "determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512"
 )
diff --git a/model_hub/Makefile b/model_hub/Makefile
@@ -5,7 +5,7 @@ SHORT_GIT_HASH := $(shell git rev-parse --short HEAD)
 ARTIFACTS_DIR := /tmp/artifacts
 
 # Model-hub library environments will be built on top of the default GPU and CPU images in master/pkg/model/defaults.go
-DEFAULT_GPU_IMAGE := determinedai/pytorch-tensorflow-cuda-dev:f20b027
+DEFAULT_GPU_IMAGE := determinedai/pytorch-tensorflow-cuda:0.35.0
 
 ############REMINDER############
 # When bumping third-party library versions, remember to bump versions in