From 4b9321285ce290aa51ecb7eb91acbb73b701d7b7 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Mon, 9 Dec 2024 21:04:23 -0800 Subject: [PATCH] add rocm 6.3 container for ci (#1429) --- azure-pipelines.yml | 2 +- .../build.sh | 2 +- .../Dockerfile | 75 +++++++++++++++++++ .../build.sh | 12 +++ .../Dockerfile | 20 +++++ .../build.sh | 15 ++++ .../docker_env_setup.sh | 7 ++ .../docker_uberenv_build.sh | 50 +++++++++++++ 8 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/build.sh create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/Dockerfile create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/build.sh create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_env_setup.sh create mode 100644 scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_uberenv_build.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 13f92279e..756e4adb8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ variables: ubuntu_22_04_tag: alpinedav/ascent-devel:ubuntu-22.04-x86_64-tpls_2024-07-10-sha3a1ef8 ubuntu_20_04_cuda_11_4_3_tag: alpinedav/ascent-devel:ubuntu-20.04-cuda-11.4.3-x86_64-tpls_2024-07-09-sha3a1ef8 ubuntu_20_04_cuda_12_1_1_tag: alpinedav/ascent-devel:ubuntu-20.04-cuda-12.1.1-x86_64-tpls_2024-07-09-sha3a1ef8 - ubuntu_20_04_rocm_6_0_0_tag: alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0--x86_64-build-ascent-tpls_2024-07-10-sha3a1ef8 + ubuntu_20_04_rocm_6_0_0_tag: alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-x86_64-build-ascent-tpls_2024-12-09-sha21f334 # only build merge target pr to develop diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.0.0_devel_build_ascent_tpls/build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.0.0_devel_build_ascent_tpls/build.sh index 81c14a90a..54cf682ac 100644 --- a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.0.0_devel_build_ascent_tpls/build.sh +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.0.0_devel_build_ascent_tpls/build.sh @@ -6,7 +6,7 @@ set -ev export REPO_NAME="ascent" export TAG_ARCH=`uname -m` -export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0--${TAG_ARCH}-build-ascent-tpls +export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0-${TAG_ARCH}-build-ascent-tpls date diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile new file mode 100644 index 000000000..26708e861 --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile @@ -0,0 +1,75 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +# other details. No copyright assignment is required to contribute to Ascent. + +FROM ubuntu:20.04 +# add sudo to base ubuntu container +# so we can install additional packages as +# non-root, but admin default user on azure pipelines +RUN apt-get update && apt-get -y install sudo +RUN apt-get update && apt-get -y install wget gnupg +# tzdata install wants to ask questions, so handled as sep case +RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata + + +# +# install rocm (recipe from warpx) +# https://github.com/ECP-WarpX/WarpX/blob/development/.github/workflows/dependencies/hip.sh +# +# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu +RUN wget -q -O - http://repo.radeon.com/rocm/rocm.gpg.key \ + | sudo apt-key add - + +RUN echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/6.3/ ubuntu main' \ + | sudo tee /etc/apt/sources.list.d/rocm.list + +RUN echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ + | sudo tee -a /etc/profile.d/rocm.sh + + + +# install std packages +# Notes: +# kmod -- hip requires lsmod +# liblocale-codes-perl -- hip uses perl, if locale bad, life isn't good +# language-pack-en -- hip uses perl, if locale bad, life isn't good +# for 6.0, `rocm` - new meta package +RUN apt-get update && apt-get -y install \ + binutils \ + gcc \ + g++ \ + gfortran \ + python \ + perl \ + git \ + git-lfs \ + curl \ + wget \ + tar \ + unzip \ + build-essential \ + libncurses-dev \ + libssl-dev \ + libblas-dev \ + liblapack-dev \ + liblapacke-dev \ + zlib1g-dev \ + libgdbm-dev \ + libreadline-dev \ + libsqlite3-dev \ + libbz2-dev \ + mpich \ + libmpich-dev \ + libnuma-dev \ + rocm-dev \ + rocfft-dev \ + rocprim-dev \ + rocrand-dev \ + hiprand-dev \ + openssh-server \ + language-pack-en \ + liblocale-codes-perl \ + vim + +#RUN apt-get update && apt-get -y install --no-install-recommends +# rocm-hip-sdk, vs all of these packages? diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/build.sh new file mode 100644 index 000000000..8f4be26af --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +# other details. No copyright assignment is required to contribute to Ascent. +set -ev + +export TAG_ARCH=`uname -m` +export TAG_NAME=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH} + +# exec docker build to create image +echo "docker build -t ${TAG_NAME} ." +docker build -t ${TAG_NAME} . diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/Dockerfile new file mode 100644 index 000000000..a37048ed8 --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/Dockerfile @@ -0,0 +1,20 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +# other details. No copyright assignment is required to contribute to Ascent. + +ARG TAG_ARCH="x86_64" +FROM alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH} + +# obtain a copy of ascent source from host env, +# which we use to call uberenv +COPY ascent.docker.src.tar.gz / +# extract +RUN tar -xzf ascent.docker.src.tar.gz + +# copy spack build script in +COPY docker_uberenv_build.sh docker_env_setup.sh / +RUN chmod -R a+x /*.sh + +RUN /docker_uberenv_build.sh + +RUN /docker_env_setup.sh diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/build.sh new file mode 100644 index 000000000..8a0be039a --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent +# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and +# other details. No copyright assignment is required to contribute to Ascent. +set -ev + +export REPO_NAME="ascent" +export TAG_ARCH=`uname -m` +export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH}-build-ascent-tpls + +date + +python3 ../../../docker_build_and_tag.py ${REPO_NAME} ${TAG_ARCH} ${TAG_BASE} + +date diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_env_setup.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_env_setup.sh new file mode 100644 index 000000000..12dfb0bb0 --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_env_setup.sh @@ -0,0 +1,7 @@ +# create some helper scripts +python ascent/scripts/gen_spack_env_script.py cmake mpi +echo "git clone --recursive https://github.com/Alpine-DAV/ascent.git" > clone.sh +chmod +x clone.sh + +# delete copy of source from host (ci will fetch new from repo) +rm -rf ascent \ No newline at end of file diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_uberenv_build.sh b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_uberenv_build.sh new file mode 100644 index 000000000..a33d67020 --- /dev/null +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_uberenv_build.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -ev + +export CMAKE_ARCH=`uname -m` + +if [[ ${CMAKE_ARCH} == "arm64" ]]; then + export CMAKE_ARCH="aarch64" +fi + +cmake_install_dir=/cmake-3.23.2-linux-${CMAKE_ARCH} +if [ ! -d ${cmake_install_dir} ]; then + # setup cmake in container + curl -L https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz -o cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz + tar -xzf cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz +fi + +export PATH=$PATH:/${cmake_install_dir}/bin/ + +# build rocm tpls with helper script +chmod +x ascent/scripts/build_ascent/build_ascent.sh +# enable tests is of b/c one of the vtk-m tests fails to link +env enable_tests=OFF build_ascent=false enable_hip=ON ascent/scripts/build_ascent/build_ascent.sh + +############################ +# TODO: get spack working +############################ +# variants +# TODO: +# (+genten) genten currently disabled, wait for genten master to gain cokurt +# (+dray+raja+umpire) (can't get raja or umpire to build with rocm) +#export SPACK_SPEC="%clang+mpi+vtkh~dray~raja~umpire+mfem+occa+rocm~genten~python~openmp~fortran" +# constraints +#export SPACK_SPEC="${SPACK_SPEC} ^hdf5~mpi ^mfem~rocm ^hypre~rocm ^conduit~fortran" +# config +#export SPACK_CONFIG="scripts/uberenv_configs/spack_configs/configs/alpinedav/ubuntu_20.04_rocm_5.1.3_devel/" + +#cd ascent && python scripts/uberenv/uberenv.py \ +# -k \ +# --spec="${SPACK_SPEC}" \ +# --spack-config-dir="${SPACK_CONFIG}" \ +# --prefix="/uberenv_libs" + +# cleanup the spack build stuff to free up space +#/uberenv_libs/spack/bin/spack clean --all + +# change perms +# chmod -R a+rX /uberenv_libs + +# back to where we started +#cd ../ \ No newline at end of file