diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index bea986153e..ce23bd295f 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -109,6 +109,11 @@ jobs: os: [el8] compiler: [gcc8, gcc9, gcc10, gcc11, icc, oneapi, nvhpc222] parallel: [serial, mpi] + include: + - os: el8 + compiler: cuda + parallel: serial + constrains: build_only steps: - uses: actions/checkout@v3 @@ -127,6 +132,7 @@ jobs: - name: Build run: gha/scripts/ci/gh-actions/run.sh build - name: Test + if: ${{ matrix.constrains != 'build_only' }} run: gha/scripts/ci/gh-actions/run.sh test linux_emu: diff --git a/scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake b/scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake new file mode 100644 index 0000000000..5855d0794e --- /dev/null +++ b/scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake @@ -0,0 +1,30 @@ +# Client maintainer: vicente.bolea@kitware.com + +include(ProcessorCount) +ProcessorCount(NCPUS) +math(EXPR N2CPUS "${NCPUS}*2") + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_SZ:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=ON +ADIOS2_USE_ZFP:BOOL=ON +ADIOS2_USE_CUDA:BOOL=ON +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall +") + +set(CTEST_TEST_ARGS PARALLEL_LEVEL ${NCPUS}) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base index 515aaa3b75..5b74cfacdc 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base @@ -1,4 +1,5 @@ -FROM ornladios/adios2:ci-spack-el8-base +ARG BASE_IMAGE=ornladios/adios2:ci-spack-el8-base +FROM $BASE_IMAGE # Install the gcc fortran compiler missing from the base image RUN dnf install -y gcc-gfortran && \ @@ -8,4 +9,5 @@ RUN dnf install -y gcc-gfortran && \ RUN . /opt/spack/share/spack/setup-env.sh && \ spack compiler rm --scope system gcc && \ spack compiler add --scope system && \ - spack config --scope system add "packages:all:compiler:[gcc]" + spack config --scope system add "packages:all:compiler:[gcc]" && \ + spack external find --scope system --not-buildable cuda diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf index d39f9d5124..8b8365cad8 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf @@ -2,39 +2,54 @@ ARG COMPILER_IMG_BASE FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base ARG EXTRA_VARIANTS -RUN sed "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \ - -i /etc/spack/packages.yaml && \ - sed "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \ - -i /etc/spack/packages.yaml +ARG CUDA_VARIANT +RUN sed \ + -e "s|packages:|packages:\n zfp:\n variants: ${CUDA_VARIANT} +shared|" \ + -e "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \ + -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \ + -i /etc/spack/packages.yaml # Build dependencies ARG COMPILER_SPACK_ID RUN . /etc/profile.d/modules.sh && \ . /opt/spack/share/spack/setup-env.sh && \ + spack spec --reuse adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \ spack install \ + --no-checksum \ + --reuse \ --fail-fast \ -v \ -j$(grep -c '^processor' /proc/cpuinfo) \ - libsodium%${COMPILER_SPACK_ID} && \ + --only dependencies \ + adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \ + spack spec --reuse libsodium%${COMPILER_SPACK_ID} && \ spack install \ + --no-checksum \ + --reuse \ --fail-fast \ -v \ -j$(grep -c '^processor' /proc/cpuinfo) \ - --only dependencies \ - adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \ + libsodium%${COMPILER_SPACK_ID} && \ spack clean -a # Setup modules RUN . /opt/spack/share/spack/setup-env.sh && \ spack env create --without-view adios2-ci && \ + spack -e adios2-ci config add concretizer:unify:true && \ spack -e adios2-ci add $(spack find --format "/{hash}") && \ + spack -e adios2-ci rm cuda && \ spack -e adios2-ci install && \ - rm -rf /root/.spack && \ spack env activate adios2-ci && \ spack env deactivate && \ + rm -rf /root/.spack && \ spack -e adios2-ci env loads # Setup default login environment -RUN echo 'source /opt/spack/share/spack/setup-env.sh' > /etc/profile.d/zz-adios2-ci-env.sh && \ - echo 'module use ${SPACK_ROOT}/share/spack/modules/linux-almalinux8-haswell' >> /etc/profile.d/zz-adios2-ci-env.sh && \ +RUN . /opt/spack/share/spack/setup-env.sh && \ + echo 'source /opt/spack/share/spack/setup-env.sh' >> /etc/profile.d/zz-adios2-ci-env.sh && \ + echo "module use ${SPACK_ROOT}/share/spack/modules/$(spack debug report | grep -Po 'Platform:\*\*\s\K.*')" >> /etc/profile.d/zz-adios2-ci-env.sh && \ echo 'source ${SPACK_ROOT}/var/spack/environments/adios2-ci/loads' >> /etc/profile.d/zz-adios2-ci-env.sh + +# Enable CUDA in Spack if enabled +COPY spack-external-cuda-addon.bash /root/spack-external-cuda-addon.bash +RUN test x"${CUDA_VARIANT}" == x"+cuda" && . /root/spack-external-cuda-addon.bash diff --git a/scripts/ci/images-v2/build-cuda.sh b/scripts/ci/images-v2/build-cuda.sh new file mode 100755 index 0000000000..62b184f300 --- /dev/null +++ b/scripts/ci/images-v2/build-cuda.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set +e + +BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE})) +source "${BASE_DIR}/build-functions.sh" + +message1 "Building ci-spack-el8 base image" +if ! build_squash \ + nvidia/cuda:11.7.0-devel-rockylinux8 \ + ornladios/adios2:ci-spack-el8-cuda-base \ + Dockerfile.ci-spack-el8-base \ + "--build-arg=BASE_IMAGE=nvidia/cuda:11.7.0-devel-rockylinux8" +then + echo "Error: Failed to build ci-spack-el8 base image" + exit 3 +fi + +message1 "Building ci-spack-el8 gcc8 base image" +if ! build_squash \ + ornladios/adios2:ci-spack-el8-cuda-base \ + ornladios/adios2:ci-spack-el8-cuda-gcc8-base \ + Dockerfile.ci-spack-el8-gcc8-base \ + "--build-arg=BASE_IMAGE=ornladios/adios2:ci-spack-el8-cuda-base" +then + echo "Error: Failed to build ci-spack-el8 leaf image" + exit 3 +fi + +message1 "Building ci-spack-el8 leaf image" +build_conf=cuda-gcc8 +if ! build_squash \ + ornladios/adios2:ci-spack-el8-cuda-gcc8-base \ + ornladios/adios2:ci-spack-el8-cuda-serial \ + Dockerfile.ci-spack-el8-leaf \ + "--build-arg=COMPILER_IMG_BASE=${build_conf} --build-arg=COMPILER_SPACK_ID=gcc --build-arg=CUDA_VARIANT=+cuda" +then + echo "Error: Failed to build ci-spack-el8 leaf image" + exit 3 +fi diff --git a/scripts/ci/images-v2/spack-external-cuda-addon.bash b/scripts/ci/images-v2/spack-external-cuda-addon.bash new file mode 100644 index 0000000000..47f26c6a61 --- /dev/null +++ b/scripts/ci/images-v2/spack-external-cuda-addon.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source /opt/spack/share/spack/setup-env.sh + +# External CUDA is erroneously treated as a module, in no CUDA builds this is no-op +find $SPACK_ROOT/share/spack/modules -type f -exec sed -i '/module load cuda/d' {} \; + +echo 'export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH' >> /etc/profile.d/zz-adios2-ci-env.sh