.gitlab-ci.yml

###################
# Global Settings #
###################

stages:
  - build_n_test

variables:
  GIT_SUBMODULE_STRATEGY: normal
  SINGULARITY_EOS_GCC_VERSION: "9.3.0"
  SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
  SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
  SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"
  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+hdf5+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
  COLOR_CYAN: "\e[1;36m"
  COLOR_PLAIN: "\e[0m"
  # set to different spack upstream for testing
  SINGULARITY_EOS_TEMP_SPACK_DIR: ""
  # uncomment for debugging the CI and persist the Spack installation
  # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
  # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"

before_script:
  - echo "Running on $(hostname)"
  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
  - export PYTHONNOUSERSITE=1
  - export SPACK_DISABLE_LOCAL_CONFIG=true
  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
  - export SINGULARITY_EOS_SPACK_DIR=${SINGULARITY_EOS_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
  - |
    rsync -rpl \
        --exclude=".git" \
        --exclude="etc/spack/repos.yaml" \
        --exclude="etc/spack/packages.yaml" \
        --include="etc/spack/**" \
        --include="lib/spack/**" \
        --exclude-from=${SINGULARITY_EOS_SPACK_DIR}/.gitignore \
        ${SINGULARITY_EOS_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
  - spack --version
  - spack compiler list
  - section end spack_init

##########################
# Build Matrix Variables #
##########################

.buildtype_debug: &buildtype_debug
  SINGULARITY_EOS_BUILD_TYPE: "Debug"

.buildtype_release: &buildtype_release
  SINGULARITY_EOS_BUILD_TYPE: "Release"
  
.buildtype_relDebugInfo: &buildtype_relDebugInfo
  SINGULARITY_EOS_BUILD_TYPE: "RelWithDebInfo"

.gpu: &gpu
  SINGULARITY_USE_CUDA: "true"
  SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+tests+cuda+kokkos+hdf5 cuda_arch=70 +kokkos-kernels%gcc@${SINGULARITY_EOS_GCC_VERSION} ^cuda@${SINGULARITY_EOS_CUDA_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"

.fort: &fort
  SINGULARITY_USE_FORTRAN: "true"

.nofort: &nofort
  SINGULARITY_USE_FORTRAN: "false"

.skylake: &skylake
    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"

.power9: &power9
    SCHEDULER_PARAMETERS: "--nodes=1 --partition=power9-rhel7"

.x86volta: &x86volta
    SCHEDULER_PARAMETERS: "--nodes=1 --partition=volta-x86 -C cpu_family:haswell"

.python: &python
    SINGULARITY_BUILD_PYTHON: "ON"

#################
# General Setup #
#################

.job: &job_def
  id_tokens:
    SITE_ID_TOKEN:
      aud: https://asc-git.lanl.gov
  tags:
    - darwin-slurm-shared
  variables:
    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
  only:
    refs:
      - main
      - merge_requests
  except:
    refs:

#######################
# Job Script Template #
#######################

.test:
  stage: build_n_test
  extends:
    - .job
  script:
    - module load gcc/${SINGULARITY_EOS_GCC_VERSION}
    - module load cuda/${SINGULARITY_EOS_CUDA_VERSION}
    - |
      if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]];
      then
        section start "spack_build[collapsed=true]" "Building via Spack"
        mkdir -p spack_build_env
        spack env create spack_build_env
        spack env activate spack_build_env
        spack repo add spack-repo
        spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
        spack dev-build -j $(nproc) ${SINGULARITY_EOS_SPACK_SPEC}
        spack env deactivate
        section end spack_build
      fi
    - section start "spack_env[collapsed=true]" "Creating Spack environment"
    - mkdir -p spack_env
    - spack env create spack_env
    - spack env activate spack_env
    - spack repo add spack-repo
    - spack config add upstreams:default:install_tree:${SINGULARITY_EOS_SPACK_DIR}/opt/spack/
    - spack spec -I ${SINGULARITY_EOS_SPACK_SPEC}
    - spack add ${SINGULARITY_EOS_SPACK_SPEC}
    - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake
    - section end spack_env
    - |
      if [[ -n ${SINGULARITY_USE_CUDA} ]];
      then
        export SINGULARITY_NVCC_WRAPPER="$(which nvcc_wrapper)";
        export SINGULARITY_CMAKE_CXX_COMPILER_FLAG="-DCMAKE_CXX_COMPILER=${SINGULARITY_NVCC_WRAPPER}";
        echo "passing '${SINGULARITY_CMAKE_CXX_COMPILER_FLAG}' to cmake"
      fi
    - mkdir -p build
    - cd build
    - |
      cmake -DCMAKE_INSTALL_PREFIX=${CI_PROJECT_DIR}/install ${SINGULARITY_CMAKE_CXX_COMPILER_FLAG} \
            -DCMAKE_BUILD_TYPE=${SINGULARITY_EOS_BUILD_TYPE} \
            -DSINGULARITY_USE_EOSPAC=ON \
            -DSINGULARITY_USE_SPINER=ON \
            -DSINGULARITY_USE_SPINER_WITH_HDF5=ON \
            -DSINGULARITY_BUILD_SESAME2SPINER=ON \
            -DSINGULARITY_BUILD_STELLARCOLLAPSE2SPINER=ON \
            -DSINGULARITY_BUILD_PYTHON=${SINGULARITY_BUILD_PYTHON:-OFF} \
            -DSINGULARITY_BUILD_TESTS=ON \
            -DSINGULARITY_BUILD_EXAMPLES=ON \
            -DSINGULARITY_TEST_SESAME=ON \
            -DSINGULARITY_TEST_STELLAR_COLLAPSE=ON \
            -DSINGULARITY_TEST_PYTHON=${SINGULARITY_BUILD_PYTHON:-OFF} \
            -DSINGULARITY_USE_CUDA=${SINGULARITY_USE_CUDA:-OFF} \
            -DSINGULARITY_USE_KOKKOS=${SINGULARITY_USE_CUDA:-OFF} \
            -DSINGULARITY_USE_KOKKOSKERNELS=${SINGULARITY_USE_CUDA:-OFF} \
            -DSINGULARITY_USE_FORTRAN=${SINGULARITY_USE_FORTRAN:-OFF} \
            ..
    - make -j $(nproc) || make VERBOSE=1
    - |
      if [[ ${CI_JOB_NAME} =~ "install" ]];
      then
        make install
        touch tests.xml
      else
        ./sesame2spiner/sesame2spiner -s materials.sp5 ../sesame2spiner/examples/unit_tests/*.dat;
        ./sesame2spiner/sesame2spiner -s duplicates.sp5 ../sesame2spiner/examples/duplicate-test/*.dat;
        export CTEST_OUTPUT_ON_FAILURE=1
        ctest --output-junit tests.xml
      fi
  artifacts:
    paths:
      - build/tests.xml
    reports:
      junit: build/tests.xml

########
# Jobs #
########

test_gnu_skylake:
  <<: *job_def
  extends: .test
  variables:
    <<: *skylake
    <<: *python

test_gnu_power9:
  <<: *job_def
  extends: .test
  variables:
    <<: *power9
    <<: *python

test_gnu_power9_gpu:
  <<: *job_def
  extends: .test
  variables:    
    <<: *power9
    <<: *buildtype_release
    <<: *gpu

test_x86_volta_gpu:
  <<: *job_def
  extends: .test
  variables:    
    <<: *x86volta
    <<: *buildtype_release
    <<: *gpu

install_gnu_skylake_fort:
  <<: *job_def
  extends: .test
  variables:
    <<: *skylake
    <<: *python
    <<: *fort

install_gnu_skylake_nofort:
  <<: *job_def
  extends: .test
  variables:
    <<: *skylake
    <<: *python
    <<: *nofort