From beaaa64e0e4c233bcc14cdad609eeacb7449fd0b Mon Sep 17 00:00:00 2001 From: PaliC Date: Mon, 17 Jun 2024 11:41:52 -0700 Subject: [PATCH 1/3] palic/update_cuda_images --- .circleci/scripts/binary_populate_env.sh | 2 +- manywheel/build_docker.sh | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.circleci/scripts/binary_populate_env.sh b/.circleci/scripts/binary_populate_env.sh index 7e663a64b..fcc21bb10 100755 --- a/.circleci/scripts/binary_populate_env.sh +++ b/.circleci/scripts/binary_populate_env.sh @@ -50,7 +50,7 @@ if [[ -z "$DOCKER_IMAGE" ]]; then elif [[ "$DESIRED_CUDA" == cpu ]]; then export DOCKER_IMAGE="pytorch/manylinux-cpu" else - export DOCKER_IMAGE="pytorch/manylinux-cuda${DESIRED_CUDA:2}" + export DOCKER_IMAGE="pytorch/manylinux-builer:${DESIRED_CUDA:2}" fi fi diff --git a/manywheel/build_docker.sh b/manywheel/build_docker.sh index 819a4a003..a7ee657bb 100755 --- a/manywheel/build_docker.sh +++ b/manywheel/build_docker.sh @@ -60,14 +60,6 @@ case ${GPU_ARCH_TYPE} in DOCKER_GPU_BUILD_ARG="" MANY_LINUX_VERSION="s390x" ;; - cuda) - TARGET=cuda_final - DOCKER_TAG=cuda${GPU_ARCH_VERSION} - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cuda${GPU_ARCH_VERSION//./} - # Keep this up to date with the minimum version of CUDA we currently support - GPU_IMAGE=centos:7 - DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9" - ;; cuda-manylinux_2_28) TARGET=cuda_final DOCKER_TAG=cuda${GPU_ARCH_VERSION} From d0791470e2d78419b7a02a996e00db60d29fd986 Mon Sep 17 00:00:00 2001 From: PaliC Date: Mon, 17 Jun 2024 11:46:42 -0700 Subject: [PATCH 2/3] [BE] Remove deprecated legacy docker image for cuda --- manywheel/build_docker.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/manywheel/build_docker.sh b/manywheel/build_docker.sh index a7ee657bb..ca7dfb98e 100755 --- a/manywheel/build_docker.sh +++ b/manywheel/build_docker.sh @@ -60,10 +60,16 @@ case ${GPU_ARCH_TYPE} in DOCKER_GPU_BUILD_ARG="" MANY_LINUX_VERSION="s390x" ;; + cuda) + TARGET=cuda_final + DOCKER_TAG=cuda${GPU_ARCH_VERSION} + # Keep this up to date with the minimum version of CUDA we currently support + GPU_IMAGE=centos:7 + DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9" + ;; cuda-manylinux_2_28) TARGET=cuda_final DOCKER_TAG=cuda${GPU_ARCH_VERSION} - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux_2_28-cuda${GPU_ARCH_VERSION//./} GPU_IMAGE=amd64/almalinux:8 DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11" MANY_LINUX_VERSION="2_28" @@ -71,7 +77,6 @@ case ${GPU_ARCH_TYPE} in cuda-aarch64) TARGET=cuda_final DOCKER_TAG=cuda${GPU_ARCH_VERSION} - LEGACY_DOCKER_IMAGE='' GPU_IMAGE=arm64v8/centos:7 DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11" MANY_LINUX_VERSION="aarch64" From 6c2cd07868e7d57ba2f1644e325da02d3100e9ab Mon Sep 17 00:00:00 2001 From: PaliC Date: Mon, 17 Jun 2024 12:11:31 -0700 Subject: [PATCH 3/3] completely remove legacy dockerfile --- .circleci/scripts/binary_populate_env.sh | 2 +- CUDA_UPGRADE_GUIDE.MD | 2 +- ffmpeg/win/Dockerfile | 2 +- manywheel/build_docker.sh | 14 -------------- 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/.circleci/scripts/binary_populate_env.sh b/.circleci/scripts/binary_populate_env.sh index fcc21bb10..560faa270 100755 --- a/.circleci/scripts/binary_populate_env.sh +++ b/.circleci/scripts/binary_populate_env.sh @@ -48,7 +48,7 @@ if [[ -z "$DOCKER_IMAGE" ]]; then if [[ "$PACKAGE_TYPE" == conda ]]; then export DOCKER_IMAGE="pytorch/conda-cuda" elif [[ "$DESIRED_CUDA" == cpu ]]; then - export DOCKER_IMAGE="pytorch/manylinux-cpu" + export DOCKER_IMAGE="pytorch/manylinux-builder:cpu" else export DOCKER_IMAGE="pytorch/manylinux-builer:${DESIRED_CUDA:2}" fi diff --git a/CUDA_UPGRADE_GUIDE.MD b/CUDA_UPGRADE_GUIDE.MD index be7bb5ff7..58897bee2 100644 --- a/CUDA_UPGRADE_GUIDE.MD +++ b/CUDA_UPGRADE_GUIDE.MD @@ -61,7 +61,7 @@ There are three types of Docker containers we maintain in order to build Linux b Add setup for our Docker `libtorch` and `manywheel`: 1. Follow this PR [PR 1003](https://github.com/pytorch/builder/pull/1003) for all steps in this section 2. For `libtorch`, the code changes are usually copy-paste. For `manywheel`, you should manually verify the versions of the shared libraries with the CUDA you downloaded before. -3. This is Manual Step: Create a ticket for PyTorch Dev Infra team to Create a new repo to host manylinux-cuda images in docker hub, for example, https://hub.docker.com/r/pytorch/manylinux-cuda115. This repo should have public visibility and read & write access for bots. This step can be removed once the following [issue](https://github.com/pytorch/builder/issues/901) is addressed. +3. This is Manual Step: Create a ticket for PyTorch Dev Infra team to Create a new repo to host manylinux-cuda images in docker hub, for example, https://hub.docker.com/r/pytorch/manylinux-builder:cuda115. This repo should have public visibility and read & write access for bots. This step can be removed once the following [issue](https://github.com/pytorch/builder/issues/901) is addressed. 4. Push the images to Docker Hub. This step should be automated with the help with GitHub Actions in the `pytorch/builder` repo. Make sure to update the `cuda_version` to the version you're adding in respective YAMLs, such as `.github/workflows/build-manywheel-images.yml`, `.github/workflows/build-conda-images.yml`, `.github/workflows/build-libtorch-images.yml`. 5. Verify that each of the workflows that push the images succeed by selecting and verifying them in the [Actions page](https://github.com/pytorch/builder/actions/workflows/build-libtorch-images.yml) of pytorch/builder. Furthermore, check [https://hub.docker.com/r/pytorch/manylinux-builder/tags](https://hub.docker.com/r/pytorch/manylinux-builder/tags), [https://hub.docker.com/r/pytorch/libtorch-cxx11-builder/tags](https://hub.docker.com/r/pytorch/libtorch-cxx11-builder/tags) to verify that the right tags exist for manylinux and libtorch types of images. 6. Finally before enabling nightly binaries and CI builds we should make sure we post following PRs in [PR 1015](https://github.com/pytorch/builder/pull/1015) [PR 1017](https://github.com/pytorch/builder/pull/1017) and [this commit](https://github.com/pytorch/builder/commit/7d5e98f1336c7cb84c772604c5e0d1acb59f2d72) to enable the new CUDA build in wheels and conda. diff --git a/ffmpeg/win/Dockerfile b/ffmpeg/win/Dockerfile index 3f6015524..d9ab1f127 100644 --- a/ffmpeg/win/Dockerfile +++ b/ffmpeg/win/Dockerfile @@ -1,5 +1,5 @@ # Base docker image for cross-compilling FFmpeg (LGPL) for Windows -FROM pytorch/manylinux-cuda101 +FROM pytorch/manylinux-builder:cuda101 COPY . /ffmpeg-build-src WORKDIR /ffmpeg-build-src diff --git a/manywheel/build_docker.sh b/manywheel/build_docker.sh index ca7dfb98e..58d74e578 100755 --- a/manywheel/build_docker.sh +++ b/manywheel/build_docker.sh @@ -16,14 +16,12 @@ case ${GPU_ARCH_TYPE} in cpu) TARGET=cpu_final DOCKER_TAG=cpu - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cpu GPU_IMAGE=centos:7 DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9" ;; cpu-manylinux_2_28) TARGET=cpu_final DOCKER_TAG=cpu - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux_2_28-cpu GPU_IMAGE=amd64/almalinux:8 DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" MANY_LINUX_VERSION="2_28" @@ -31,7 +29,6 @@ case ${GPU_ARCH_TYPE} in cpu-aarch64) TARGET=final DOCKER_TAG=cpu-aarch64 - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cpu-aarch64 GPU_IMAGE=arm64v8/centos:7 DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=10" MANY_LINUX_VERSION="aarch64" @@ -39,7 +36,6 @@ case ${GPU_ARCH_TYPE} in cpu-aarch64-2_28) TARGET=final DOCKER_TAG=cpu-aarch64 - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux_2_28-cpu-aarch64 GPU_IMAGE=arm64v8/almalinux:8 DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" MANY_LINUX_VERSION="2_28_aarch64" @@ -47,7 +43,6 @@ case ${GPU_ARCH_TYPE} in cpu-cxx11-abi) TARGET=final DOCKER_TAG=cpu-cxx11-abi - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cpu-cxx11-abi GPU_IMAGE="" DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9" MANY_LINUX_VERSION="cxx11-abi" @@ -55,7 +50,6 @@ case ${GPU_ARCH_TYPE} in cpu-s390x) TARGET=final DOCKER_TAG=cpu-s390x - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cpu-s390x GPU_IMAGE=redhat/ubi9 DOCKER_GPU_BUILD_ARG="" MANY_LINUX_VERSION="s390x" @@ -85,7 +79,6 @@ case ${GPU_ARCH_TYPE} in rocm) TARGET=rocm_final DOCKER_TAG=rocm${GPU_ARCH_VERSION} - LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-rocm:${GPU_ARCH_VERSION} GPU_IMAGE=rocm/dev-centos-7:${GPU_ARCH_VERSION}-complete PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100" ROCM_REGEX="([0-9]+)\.([0-9]+)[\.]?([0-9]*)" @@ -111,7 +104,6 @@ DOCKER_NAME=manylinux${MANY_LINUX_VERSION} DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/${DOCKER_NAME}-builder:${DOCKER_TAG} if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION} - LEGACY_DOCKER_IMAGE='' fi ( set -x @@ -132,9 +124,6 @@ DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA} ( set -x - if [[ -n ${LEGACY_DOCKER_IMAGE} ]]; then - docker tag ${DOCKER_IMAGE} ${LEGACY_DOCKER_IMAGE} - fi if [[ -n ${GITHUB_REF} ]]; then docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG} docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG} @@ -145,9 +134,6 @@ if [[ "${WITH_PUSH}" == true ]]; then ( set -x docker push "${DOCKER_IMAGE}" - if [[ -n ${LEGACY_DOCKER_IMAGE} ]]; then - docker push "${LEGACY_DOCKER_IMAGE}" - fi if [[ -n ${GITHUB_REF} ]]; then docker push "${DOCKER_IMAGE_BRANCH_TAG}" docker push "${DOCKER_IMAGE_SHA_TAG}"