CI: add CUDA build_only build

ornladios · Jul 15, 2022 · 7c05bda · 7c05bda
1 parent 9431745
commit 7c05bda
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 12 deletions.
diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml
@@ -109,6 +109,11 @@ jobs:
         os: [el8]
         compiler: [gcc8, gcc9, gcc10, gcc11, icc, oneapi, nvhpc222]
         parallel: [serial, mpi]
+        include:
+          - os: el8
+            compiler: cuda
+            parallel: serial
+            constrains: build_only
 
     steps:
       - uses: actions/checkout@v3
@@ -127,6 +132,7 @@ jobs:
       - name: Build
         run: gha/scripts/ci/gh-actions/run.sh build
       - name: Test
+        if: ${{ matrix.constrains != 'build_only' }}
         run: gha/scripts/ci/gh-actions/run.sh test
 
   linux_emu:

diff --git a/scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake b/scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake
@@ -0,0 +1,30 @@
+# Client maintainer: [email protected]
+
+include(ProcessorCount)
+ProcessorCount(NCPUS)
+math(EXPR N2CPUS "${NCPUS}*2")
+
+set(ENV{CC}  gcc)
+set(ENV{CXX} g++)
+set(ENV{FC}  gfortran)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=ON
+ADIOS2_USE_Blosc:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_HDF5:BOOL=ON
+ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
+ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
+ADIOS2_USE_CUDA:BOOL=ON
+CMAKE_C_FLAGS:STRING=-Wall
+CMAKE_CXX_FLAGS:STRING=-Wall
+CMAKE_Fortran_FLAGS:STRING=-Wall
+")
+
+set(CTEST_TEST_ARGS PARALLEL_LEVEL ${NCPUS})
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base
@@ -1,4 +1,5 @@
-FROM ornladios/adios2:ci-spack-el8-base
+ARG BASE_IMAGE=ornladios/adios2:ci-spack-el8-base
+FROM $BASE_IMAGE
 
 # Install the gcc fortran compiler missing from the base image
 RUN dnf install -y gcc-gfortran && \
@@ -8,4 +9,5 @@ RUN dnf install -y gcc-gfortran && \
 RUN . /opt/spack/share/spack/setup-env.sh && \
     spack compiler rm --scope system gcc && \
     spack compiler add --scope system && \
-    spack config --scope system add "packages:all:compiler:[gcc]"
+    spack config --scope system add "packages:all:compiler:[gcc]" && \
+    spack external find --scope system --not-buildable cuda
diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf
@@ -2,39 +2,54 @@ ARG COMPILER_IMG_BASE
 FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base
 
 ARG EXTRA_VARIANTS
-RUN sed "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \
-      -i /etc/spack/packages.yaml && \
-    sed "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \
-      -i /etc/spack/packages.yaml
+ARG CUDA_VARIANT
+RUN sed \
+    -e "s|packages:|packages:\n  zfp:\n    variants: ${CUDA_VARIANT} +shared|" \
+    -e "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \
+    -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \
+    -i /etc/spack/packages.yaml
 
 # Build dependencies
 ARG COMPILER_SPACK_ID
 RUN . /etc/profile.d/modules.sh && \
     . /opt/spack/share/spack/setup-env.sh && \
+    spack spec --reuse adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
     spack install \
+      --no-checksum \
+      --reuse \
       --fail-fast \
       -v \
       -j$(grep -c '^processor' /proc/cpuinfo) \
-      libsodium%${COMPILER_SPACK_ID} && \
+      --only dependencies \
+      adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
+    spack spec --reuse libsodium%${COMPILER_SPACK_ID} && \
     spack install \
+      --no-checksum \
+      --reuse \
       --fail-fast \
       -v \
       -j$(grep -c '^processor' /proc/cpuinfo) \
-      --only dependencies \
-      adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
+      libsodium%${COMPILER_SPACK_ID} && \
     spack clean -a
 
 # Setup modules
 RUN . /opt/spack/share/spack/setup-env.sh && \
     spack env create --without-view adios2-ci && \
+    spack -e adios2-ci config add concretizer:unify:true && \
     spack -e adios2-ci add $(spack find --format "/{hash}") && \
+    spack -e adios2-ci rm cuda && \
     spack -e adios2-ci install && \
-    rm -rf /root/.spack && \
     spack env activate adios2-ci && \
     spack env deactivate && \
+    rm -rf /root/.spack && \
     spack -e adios2-ci env loads
 
 # Setup default login environment
-RUN echo 'source /opt/spack/share/spack/setup-env.sh' > /etc/profile.d/zz-adios2-ci-env.sh && \
-    echo 'module use ${SPACK_ROOT}/share/spack/modules/linux-almalinux8-haswell' >> /etc/profile.d/zz-adios2-ci-env.sh && \
+RUN . /opt/spack/share/spack/setup-env.sh && \
+    echo 'source /opt/spack/share/spack/setup-env.sh' >> /etc/profile.d/zz-adios2-ci-env.sh && \
+    echo "module use ${SPACK_ROOT}/share/spack/modules/$(spack debug report | grep -Po 'Platform:\*\*\s\K.*')" >> /etc/profile.d/zz-adios2-ci-env.sh && \
     echo 'source ${SPACK_ROOT}/var/spack/environments/adios2-ci/loads' >> /etc/profile.d/zz-adios2-ci-env.sh
+
+# Enable CUDA in Spack if enabled
+COPY spack-external-cuda-addon.bash /root/spack-external-cuda-addon.bash
+RUN test x"${CUDA_VARIANT}" == x"+cuda" && . /root/spack-external-cuda-addon.bash
diff --git a/scripts/ci/images-v2/build-cuda.sh b/scripts/ci/images-v2/build-cuda.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set +e
+
+BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE}))
+source "${BASE_DIR}/build-functions.sh"
+
+message1 "Building ci-spack-el8 base image"
+if ! build_squash \
+    nvidia/cuda:11.7.0-devel-rockylinux8 \
+    ornladios/adios2:ci-spack-el8-cuda-base \
+    Dockerfile.ci-spack-el8-base \
+    "--build-arg=BASE_IMAGE=nvidia/cuda:11.7.0-devel-rockylinux8"
+then
+  echo "Error: Failed to build ci-spack-el8 base image"
+  exit 3
+fi
+
+message1 "Building ci-spack-el8 gcc8 base image"
+if ! build_squash \
+    ornladios/adios2:ci-spack-el8-cuda-base \
+    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
+    Dockerfile.ci-spack-el8-gcc8-base \
+    "--build-arg=BASE_IMAGE=ornladios/adios2:ci-spack-el8-cuda-base"
+then
+  echo "Error: Failed to build ci-spack-el8 leaf image"
+  exit 3
+fi
+
+message1 "Building ci-spack-el8 leaf image"
+build_conf=cuda-gcc8
+if ! build_squash \
+    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
+    ornladios/adios2:ci-spack-el8-cuda-serial \
+    Dockerfile.ci-spack-el8-leaf \
+    "--build-arg=COMPILER_IMG_BASE=${build_conf} --build-arg=COMPILER_SPACK_ID=gcc --build-arg=CUDA_VARIANT=+cuda"
+then
+  echo "Error: Failed to build ci-spack-el8 leaf image"
+  exit 3
+fi
diff --git a/scripts/ci/images-v2/spack-external-cuda-addon.bash b/scripts/ci/images-v2/spack-external-cuda-addon.bash
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+source /opt/spack/share/spack/setup-env.sh
+
+# External CUDA is erroneously treated as a module, in no CUDA builds this is no-op
+find $SPACK_ROOT/share/spack/modules -type f -exec sed -i '/module load cuda/d' {} \;
+
+echo 'export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH' >> /etc/profile.d/zz-adios2-ci-env.sh