add nccl, cvmfs, cuda-mpi modules, update mpich and gpu modules

etc/01-cvmfs.conf

@@ -0,0 +1,2 @@
+/cvmfs
+

etc/01-nccl.conf

@@ -0,0 +1,3 @@
+/opt/udiImage/modules/nccl-2.15/lib
+/opt/udiImage/modules/nccl-2.15/deps/lib
+

etc/modules.d/cuda-mpich.yaml

@@ -0,0 +1,28 @@
+name: cuda-mpi
+cli_arg: cuda-mpi
+help: Enable CUDA-aware Cray MPICH Support
+env: ENABLE_CUDA_MPICH_SS
+shared_run: True
+additional_args:
+  - -e SLURM_*
+  - -e PALS_*
+  - -e PMI_*
+  - -e LD_LIBRARY_PATH=/opt/udiImage/modules/mpich:/opt/udiImage/modules/mpich/dep
+  - -e MPICH_GPU_SUPPORT_ENABLED=1
+  - -e LD_PRELOAD=/opt/udiImage/modules/mpich/libmpi_gtl_cuda.so.0
+  - --ipc=host
+  - --network=host
+  - --pid=host
+  - --privileged
+copy:
+  - ../01-mpich.conf:/etc/ld.so.conf.d/02-mpich.conf
+bind:
+  - /dev/xpmem:/dev/xpmem
+  - /dev/shm:/dev/shm
+  - /dev/ss0:/dev/ss0
+  - /dev/cxi*:/dev/
+  - /usr/lib/shifter/mpich-2.2:/opt/udiImage/modules/mpich
+  - /var/spool/slurmd:/var/spool/slurmd
+  - /run/munge:/run/munge
+  - /run/nscd:/run/nscd
+  - /etc/libibverbs.d:/etc/libibverbs.d

etc/modules.d/cvmfs.yaml

@@ -0,0 +1,8 @@
+name: cvmfs
+cli_arg: cvmfs
+help: Enable cmvfs filesystem
+env: ENABLE_CVMFS
+copy:
+  - ../01-cvmfs.conf:/etc/ld.so.conf.d/02-cvmfs.conf
+bind:
+  - /cvmfs:/cvmfs

etc/modules.d/gpu.yaml

@@ -5,14 +5,14 @@ env: ENABLE_GPU
 additional_args:
   - -e NVIDIA_VISIBLE_DEVICES
 copy:
-  - ../01-gpu.conf:/etc/ld.so.conf.d/01-gpu.conf
-  - /usr/lib64/libnv*:/usr/lib64/libnv*
-  - /usr/lib64/nvidia/libOpenCL*:/usr/lib64/libOpenCL*
-  - /usr/lib64/libcuda*:/usr/lib64/libcuda*
-  - /opt/cray/pe/mpich/default/gtl/lib/libmpi_gtl_cuda*:/usr/lib64/libmpi_gtl_cuda*
+  - ../01-gpu.conf:/etc/ld.so.conf.d/02-gpu.conf
+  - /usr/lib64/libnv*:/usr/lib64/
+  - /usr/lib64/nvidia/libOpenCL*:/usr/lib64/
+  - /usr/lib64/libcuda*:/usr/lib64/
+  - /opt/cray/pe/mpich/default/gtl/lib/libmpi_gtl_cuda*:/usr/lib64/
 bind:
   - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi
   - /dev/nvidiactl:/dev/nvidiactl
-  - /dev/nvidia*:/dev/nvidia*
+  - /dev/nvidia*:/dev/
   - /dev/nvidia-uvm:/dev/nvidia-uvm
   - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools

etc/modules.d/mpich.yaml

@@ -18,9 +18,8 @@ bind:
   - /dev/xpmem:/dev/xpmem
   - /dev/shm:/dev/shm
   - /dev/ss0:/dev/ss0
-  - /dev/cxi*:/dev/cxi*
+  - /dev/cxi*:/dev/
   - /usr/lib/shifter/mpich-2.2:/opt/udiImage/modules/mpich
-  - /opt/cray/libfabric/*/lib64/libfabric:/opt/cray/libfabric/*/lib64/libfabric
   - /var/spool/slurmd:/var/spool/slurmd
   - /run/munge:/run/munge
   - /run/nscd:/run/nscd

etc/modules.d/nccl.yaml

@@ -0,0 +1,40 @@
+name: nccl
+cli_arg: nccl
+help: Enable SS11 support for NCCL
+env: ENABLE_NCCL
+additional_args:
+  - -e LD_LIBRARY_PATH=/opt/udiImage/modules/nccl-2.15/lib:/opt/udiImage/modules/nccl-2.15/deps/lib
+  - -e FI_CXI_DISABLE_HOST_REGISTER=1
+  - -e NCCL_CROSS_NIC=1
+  - -e NCCL_SOCKET_IFNAME=hsn
+  - -e CCL_NET_GDR_LEVEL=PHB
+  - -e NCCL_NET="AWS Libfabric"
+copy:
+  - ../01-nccl.conf:/etc/ld.so.conf.d/02-nccl.conf
+  - /global/common/software/nersc/pm-2022q4/sw/evp-patch/lib64/libcrypto.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /global/common/software/nersc/pm-2022q4/sw/evp-patch/lib64/libssh.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /global/common/software/nersc/pm-2022q4/sw/evp-patch/lib64/libssl.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /global/common/software/nersc/pm-2022q4/sw/nccl-2.15.5-ofi-r4/lib/libnccl*:/opt/udiImage/modules/nccl-2.15/lib/
+  - /global/common/software/nersc/pm-2022q4/sw/nccl-2.15.5-ofi-r4/include/*:/opt/udiImage/modules/nccl-2.15/include/
+  - /usr/lib64/libcxi.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libcurl.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libjson-c.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /opt/cray/pe/gcc/11.2.0/snos/lib64/libatomic.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /opt/cray/libfabric/1.15.2.0/lib64/libfabric.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /opt/cray/pe/mpich/8.1.22/gtl/lib/libmpi_gtl_cuda.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /opt/cray/xpmem/default/lib64/libxpmem.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libnghttp2.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libidn2.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libpsl.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libgssapi_krb5.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libldap_r-2.4.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/liblber-2.4.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libunistring.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libkrb5.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libk5crypto.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libcom_err.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libkrb5support.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libresolv.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /lib64/libresolv*:/lib64/
+  - /usr/lib64/libsasl2.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/
+  - /usr/lib64/libkeyutils.so*:/opt/udiImage/modules/nccl-2.15/deps/lib/

setup.cfg

@@ -20,8 +20,13 @@ console_scripts =
     etc/podman_hpc =
       etc/01-gpu.conf
       etc/01-mpich.conf
+      etc/01-nccl.conf
+      etc/01-cvmfs.conf
       etc/podman_hpc.yaml
     etc/podman_hpc/modules.d =
       etc/modules.d/gpu.yaml
       etc/modules.d/mpich.yaml
+      etc/modules/d/cuda-mpich.yaml
+      etc/modules.d/nccl.yaml
+      etc/modules.d/cvmfs.yaml