CI: adds CUDA build

.github/workflows/everything.yml

@@ -109,6 +109,11 @@ jobs:
         os: [el8]
         compiler: [gcc8, gcc9, gcc10, gcc11, icc, oneapi, nvhpc222]
         parallel: [serial, mpi]
+        include:
+          - os: el8
+            compiler: cuda
+            parallel: serial
+            constrains: build_only
 
     steps:
       - uses: actions/checkout@v3
@@ -127,6 +132,7 @@ jobs:
       - name: Build
         run: gha/scripts/ci/gh-actions/run.sh build
       - name: Test
+        if: ${{ matrix.constrains != 'build_only' }}
         run: gha/scripts/ci/gh-actions/run.sh test
 
   linux_emu:

scripts/ci/cmake-v2/ci-el8-cuda-serial.cmake

@@ -0,0 +1,30 @@
+# Client maintainer: [email protected]
+
+include(ProcessorCount)
+ProcessorCount(NCPUS)
+math(EXPR N2CPUS "${NCPUS}*2")
+
+set(ENV{CC}  gcc)
+set(ENV{CXX} g++)
+set(ENV{FC}  gfortran)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=ON
+ADIOS2_USE_Blosc:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_HDF5:BOOL=ON
+ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
+ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
+ADIOS2_USE_CUDA:BOOL=ON
+CMAKE_C_FLAGS:STRING=-Wall
+CMAKE_CXX_FLAGS:STRING=-Wall
+CMAKE_Fortran_FLAGS:STRING=-Wall
+")
+
+set(CTEST_TEST_ARGS PARALLEL_LEVEL ${NCPUS})
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/images-v2/0001-spack-rhel-fix.patch

@@ -0,0 +1,11 @@
+--- lib/spack/spack/solver/concretize.lp
++++ lib/spack/spack/solver/concretize.lp
+@@ -718,6 +718,9 @@
+ % catalina binaries can be used on bigsur. Direction is package -> dependency.
+ os_compatible("bigsur", "catalina").
+
++os_compatible("rocky8", "centos8").
++os_compatible("centos8", "rocky8").
++
+ % If an OS is set explicitly respect the value
+ node_os(Package, OS) :- node_os_set(Package, OS), node(Package).

scripts/ci/images-v2/Dockerfile.ci-spack-el8-base

@@ -1,4 +1,5 @@
-FROM almalinux/almalinux:8
+ARG BASE_IMAGE=almalinux/almalinux:8
+FROM $BASE_IMAGE
 
 RUN dnf upgrade -y && \
     dnf install -y glibc-langpack-en
@@ -11,12 +12,19 @@ RUN dnf install -y \
         python3 python3-pip
 RUN dnf clean all
 
-RUN git clone https://github.com/spack/spack.git /opt/spack
+RUN git clone -b v0.18.0 https://github.com/spack/spack.git /opt/spack
 COPY packages.yaml.base /etc/spack/packages.yaml
 COPY modules.yaml /etc/spack/modules.yaml
+COPY 0001-spack-rhel-fix.patch /root/0001-spack-rhel-fix.patch
+
+RUN cd /opt/spack && \
+    patch -p0 < /root/0001-spack-rhel-fix.patch
 
 RUN . /opt/spack/share/spack/setup-env.sh && \
+    spack config --scope system add concretizer:targets:granularity:generic && \
+    spack config update modules -y && \
     spack compiler find --scope system && \
-    spack spec zlib && \
     spack clean -a && \
+    spack mirror add E4S https://cache.e4s.io && \
+    spack buildcache keys -it && \
     spack spec zlib

scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base

@@ -1,4 +1,5 @@
-FROM ornladios/adios2:ci-spack-el8-base
+ARG BASE_IMAGE=ornladios/adios2:ci-spack-el8-base
+FROM $BASE_IMAGE
 
 # Install the gcc fortran compiler missing from the base image
 RUN dnf install -y gcc-gfortran && \
@@ -8,4 +9,5 @@ RUN dnf install -y gcc-gfortran && \
 RUN . /opt/spack/share/spack/setup-env.sh && \
     spack compiler rm --scope system gcc && \
     spack compiler add --scope system && \
-    spack config --scope system add "packages:all:compiler:[gcc]"
+    spack config --scope system add "packages:all:compiler:[gcc]" && \
+    spack external find --scope system --not-buildable cuda

scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf

@@ -2,39 +2,54 @@ ARG COMPILER_IMG_BASE
 FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base
 
 ARG EXTRA_VARIANTS
-RUN sed "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \
-      -i /etc/spack/packages.yaml && \
-    sed "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \
-      -i /etc/spack/packages.yaml
+ARG CUDA_VARIANT
+RUN sed \
+    -e "s|packages:|packages:\n  zfp:\n    variants: ${CUDA_VARIANT} +shared|" \
+    -e "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \
+    -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \
+    -i /etc/spack/packages.yaml
 
 # Build dependencies
 ARG COMPILER_SPACK_ID
 RUN . /etc/profile.d/modules.sh && \
     . /opt/spack/share/spack/setup-env.sh && \
+    spack spec --reuse adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
     spack install \
+      --no-checksum \
+      --reuse \
       --fail-fast \
       -v \
       -j$(grep -c '^processor' /proc/cpuinfo) \
-      libsodium%${COMPILER_SPACK_ID} && \
+      --only dependencies \
+      adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
+    spack spec --reuse libsodium%${COMPILER_SPACK_ID} && \
     spack install \
+      --no-checksum \
+      --reuse \
       --fail-fast \
       -v \
       -j$(grep -c '^processor' /proc/cpuinfo) \
-      --only dependencies \
-      adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \
+      libsodium%${COMPILER_SPACK_ID} && \
     spack clean -a
 
 # Setup modules
 RUN . /opt/spack/share/spack/setup-env.sh && \
     spack env create --without-view adios2-ci && \
+    spack -e adios2-ci config add concretizer:unify:true && \
     spack -e adios2-ci add $(spack find --format "/{hash}") && \
+    spack -e adios2-ci rm cuda && \
     spack -e adios2-ci install && \
-    rm -rf /root/.spack && \
     spack env activate adios2-ci && \
     spack env deactivate && \
+    rm -rf /root/.spack && \
     spack -e adios2-ci env loads
 
 # Setup default login environment
-RUN echo 'source /opt/spack/share/spack/setup-env.sh' > /etc/profile.d/zz-adios2-ci-env.sh && \
-    echo 'module use ${SPACK_ROOT}/share/spack/modules/linux-almalinux8-haswell' >> /etc/profile.d/zz-adios2-ci-env.sh && \
+RUN . /opt/spack/share/spack/setup-env.sh && \
+    echo 'source /opt/spack/share/spack/setup-env.sh' >> /etc/profile.d/zz-adios2-ci-env.sh && \
+    echo "module use ${SPACK_ROOT}/share/spack/modules/$(spack debug report | grep -Po 'Platform:\*\*\s\K.*')" >> /etc/profile.d/zz-adios2-ci-env.sh && \
     echo 'source ${SPACK_ROOT}/var/spack/environments/adios2-ci/loads' >> /etc/profile.d/zz-adios2-ci-env.sh
+
+# Enable CUDA in Spack if enabled
+COPY spack-external-cuda-addon.bash /root/spack-external-cuda-addon.bash
+RUN test x"${CUDA_VARIANT}" == x"+cuda" && . /root/spack-external-cuda-addon.bash

scripts/ci/images-v2/build-cuda.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set +e
+
+BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE}))
+source "${BASE_DIR}/build-functions.sh"
+
+message1 "Building ci-spack-el8 base image"
+if ! build_squash \
+    nvidia/cuda:11.7.0-devel-rockylinux8 \
+    ornladios/adios2:ci-spack-el8-cuda-base \
+    Dockerfile.ci-spack-el8-base \
+    "--build-arg=BASE_IMAGE=nvidia/cuda:11.7.0-devel-rockylinux8"
+then
+  echo "Error: Failed to build ci-spack-el8 base image"
+  exit 3
+fi
+
+message1 "Building ci-spack-el8 gcc8 base image"
+if ! build_squash \
+    ornladios/adios2:ci-spack-el8-cuda-base \
+    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
+    Dockerfile.ci-spack-el8-gcc8-base \
+    "--build-arg=BASE_IMAGE=ornladios/adios2:ci-spack-el8-cuda-base"
+then
+  echo "Error: Failed to build ci-spack-el8 leaf image"
+  exit 3
+fi
+
+message1 "Building ci-spack-el8 leaf image"
+build_conf=cuda-gcc8
+if ! build_squash \
+    ornladios/adios2:ci-spack-el8-cuda-gcc8-base \
+    ornladios/adios2:ci-spack-el8-cuda-serial \
+    Dockerfile.ci-spack-el8-leaf \
+    "--build-arg=COMPILER_IMG_BASE=${build_conf} --build-arg=COMPILER_SPACK_ID=gcc --build-arg=CUDA_VARIANT=+cuda"
+then
+  echo "Error: Failed to build ci-spack-el8 leaf image"
+  exit 3
+fi

scripts/ci/images-v2/packages.yaml.base

@@ -1,6 +1,6 @@
 packages:
   all:
-    target: [haswell]
+    target: [x86_64]
   cmake:
     variants: ~ownlibs
   adios2:

scripts/ci/images-v2/spack-external-cuda-addon.bash

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+source /opt/spack/share/spack/setup-env.sh
+
+# External CUDA is erroneously treated as a module, in no CUDA builds this is no-op
+find $SPACK_ROOT/share/spack/modules -type f -exec sed -i '/module load cuda/d' {} \;
+
+echo 'export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH' >> /etc/profile.d/zz-adios2-ci-env.sh

testing/adios2/engine/bp/TestBPWriteReadCuda.cpp

@@ -140,11 +140,11 @@ void RateCUDA(const std::string mode)
                            std::bind(std::minus<float>(), std::placeholders::_1,
                                      INCREMENT));
 
-            for (int i = 0; i < NxTotal; i++)
+            for (size_t i = 0; i < NxTotal; i++)
             {
                 char msg[1 << 8] = {0};
-                sprintf(msg, "t=%d i=%d rank=%d r32o=%f r32s=%f", t, i, mpiRank,
-                        r32o[i], r32s[i]);
+                sprintf(msg, "t=%d i=%zu rank=%d r32o=%f r32s=%f", t, i,
+                        mpiRank, r32o[i], r32s[i]);
                 ASSERT_LT(std::abs(r32o[i] - r32s[i]), EPSILON) << msg;
             }
         }

testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp

@@ -125,11 +125,11 @@ void ZFPRateCUDA(const std::string rate)
                            std::bind(std::minus<float>(), std::placeholders::_1,
                                      INCREMENT));
 
-            for (int i = 0; i < NxTotal; i++)
+            for (size_t i = 0; i < NxTotal; i++)
             {
                 char msg[1 << 8] = {0};
-                sprintf(msg, "t=%d i=%d rank=%d r32o=%f r32s=%f", t, i, mpiRank,
-                        r32o[i], r32s[i]);
+                sprintf(msg, "t=%d i=%zu rank=%d r32o=%f r32s=%f", t, i,
+                        mpiRank, r32o[i], r32s[i]);
                 ASSERT_LT(std::abs(r32o[i] - r32s[i]), EPSILON) << msg;
             }
         }