.gitlab-ci.yml

# SPDX-FileCopyrightText: 2017-2023 CERN and the Allpix Squared authors
# SPDX-License-Identifier: MIT

variables:
    REPOSITORY: "https://gitlab.cern.ch/allpix-squared/allpix-squared.git"
    DOCKER_DEPS_IMAGE: "gitlab-registry.cern.ch/allpix-squared/allpix-squared/allpix-squared-deps:v6"
    EOS_PATH: "/eos/project/a/allpix-squared/www/"
    DOCKER_FILE: etc/docker/Dockerfile

default:
    retry:
        max: 2
        when:
            - stuck_or_timeout_failure
            - runner_system_failure

stages:
    - compilation
    - testing
    - formatting
    - performance
    - documentation
    - packaging
    - deployment


############################
# General target templates #
############################

.centos7:
    image: gitlab-registry.cern.ch/sft/docker/centos7:latest
    tags:
        - cvmfs
    variables:
        COMPILER_TYPE: "gcc"
    before_script:
        - source .ci/init_x86_64.sh
        - mkdir -p build/ && cd build/

.alma9:
    image: gitlab-registry.cern.ch/sft/docker/alma9:latest
    tags:
        - cvmfs
    variables:
        COMPILER_TYPE: "gcc"
    before_script:
        - source .ci/init_x86_64.sh
        - mkdir -p build/ && cd build/

.docker:
    image: $DOCKER_DEPS_IMAGE
    tags:
        - k8s-default
    before_script:
        - mkdir -p build/ && cd build/

.lxplus:
    image: gitlab-registry.cern.ch/sft/docker/alma9:latest
    tags:
        - cvmfs
    variables:
        COMPILER_TYPE: "gcc"
    before_script:
        - source etc/scripts/setup_lxplus.sh
        - mkdir -p build/ && cd build/


#######################
# Compilation targets #
#######################

# Hidden key to define the default compile job:
.compile:
    variables:
        ALLPIX_BUILD_ENV: "CI/CD"
    stage: compilation
    needs: []
    script:
        - export CCACHE_DIR=`pwd`/ccache
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - ninja -k0
        - ninja install
    artifacts:
        paths:
            - build
            - bin
            - lib
            - share
        expire_in: 24 hour

# Ensure that our LXPLUS setup script is functional:
cmp:lxplus-gcc:
    extends:
        - .lxplus
        - .compile
    script:
        - export CCACHE_DIR=`pwd`/ccache
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - ninja -k0
        - ninja install

cmp:docker:
    extends:
        - .docker
        - .compile
    script:
        - cmake -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - make
        - make install

cmp:cc7-gcc:
    extends:
        - .centos7
        - .compile

cmp:el9-llvm:
    extends:
        - .alma9
        - .compile
    variables:
        COMPILER_TYPE: "llvm"

cmp:el9-gcc:
    extends:
        - .alma9
        - .compile


############################
# Format and Lint Checking #
############################

# Hidden key to define the basis for linting and formatting:
.format:
    stage: formatting
    needs: []
    dependencies: []
    extends: .alma9
    variables:
        COMPILER_TYPE: "llvm"

fmt:clang-format:
    extends: .format
    script:
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - ninja check-format

fmt:clang-lint:
    extends: .format
    script:
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - ninja check-lint 2> /dev/null
    only:
        - schedules
        - tags

fmt:clang-lint-diff:
    extends: .format
    script:
        - git remote add upstream $REPOSITORY && git fetch upstream
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE -DTARGET_REMOTE="upstream" ..
        - ninja check-lint-diff 2> /dev/null

fmt:cmake-lint:
    extends: .format
    script:
        - export PATH=$PATH:~/.local/bin
        - pip install --trusted-host=pypi.org --user cmakelang
        - cmake -GNinja ..
        - ninja lint-cmake

fmt:coverage:
    extends: .format
    variables:
        COMPILER_TYPE: "gcc"
    script:
        - export PATH=$PATH:~/.local/bin
        - export PYTHONPATH=~/.local/lib/python3.9/site-packages:$PYTHONPATH
        - pip install --trusted-host=pypi.org --user gcovr==5.0
        - export CCACHE_DIR=`pwd`/ccache
        - cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE=ON ..
        - ninja -k0
        - ninja install
        - ninja coverage_gcovr

fmt:codespell:
    extends: .format
    script:
        - export PATH=~/.local/bin:$PATH
        - cd ../
        - pip install --trusted-host=pypi.org --user codespell
        - codespell --ignore-words .gitlab/ci/codespell_ignored_words.txt --quiet-level 2 --skip ".git,.gitlab,cmake,3rdparty,AUTHORS.md"

fmt:proselint:
    extends: .format
    script:
        - export PATH=~/.local/bin:$PATH
        - cd ../
        - pip install --trusted-host=pypi.org --user proselint
        - find . -name "*md" -not -path "./build/*" -print0 | xargs -0 proselint --config .proselint.json

fmt:coverity:
    extends: .format
    only:
        - schedules
    variables:
        COMPILER_TYPE: "gcc"
        SCHEDULE_TAG: "coverity"
    script:
        - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/cov-analysis-linux64.tgz
        - tar xfz /tmp/cov-analysis-linux64.tgz
        - export CCACHE_DIR=`pwd`/ccache
        - cmake -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_ALL_MODULES=ON -DCMAKE_BUILD_TYPE=RELEASE ..
        - cov-analysis-linux64-*/bin/cov-build --dir cov-int make
        - tar cfz cov-int.tar.gz cov-int
        - curl --form token=$COVERITY_SCAN_TOKEN --form email=$GITLAB_USER_EMAIL --form file=@cov-int.tar.gz --form version="`git describe --tags`" --form description="`git describe --tags` / $CI_COMMIT_TITLE / $CI_COMMIT_REF_NAME:$CI_PIPELINE_ID " https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME

fmt:reuse:
    image:
        name: registry.cern.ch/docker.io/fsfe/reuse:latest
        entrypoint: [""]
    tags:
        - k8s-default
    stage: formatting
    needs: []
    dependencies: []
    script:
        - reuse lint

##############
# Unit tests #
##############

.test:
    stage: testing
    after_script:
        - source .ci/init_x86_64.sh
        - .gitlab/ci/transform_ctest_junit.py build/Testing/`head -n 1 build/Testing/TAG`/Test.xml .gitlab/ci/ctest-to-junit.xsl apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml
    artifacts:
        when: always
        expire_in: 1 week
        name: "apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
        paths:
            - apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml
        reports:
            junit: apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml

.testmod:
    extends: .test
    script:
        - ctest -R modules/ --repeat until-pass:2 --no-compress-output --test-action Test

.testcore:
    extends: .test
    script:
        - ctest -R core/ --repeat until-pass:2 --no-compress-output --test-action Test


# Test if examples still execute fine:
examples:
    extends:
        - .centos7
        - .test
    needs:
        - job: cmp:cc7-gcc
          artifacts: true
    script:
        - ctest -R examples/ --repeat until-pass:2 --no-compress-output --test-action Test

# LXPLUS
mod:lxplus-gcc:
    extends:
        - .lxplus
        - .testmod
    needs:
        - job: cmp:lxplus-gcc
          artifacts: true

core:lxplus-gcc:
    extends:
        - .lxplus
        - .testcore
    needs:
        - job: cmp:lxplus-gcc
          artifacts: true

# Ubuntu Docker image - overriding after_script to not load CVMFS
mod:docker:
    extends:
        - .docker
        - .testmod
    needs:
        - job: cmp:docker
          artifacts: true
    after_script:
        - .gitlab/ci/transform_ctest_junit.py build/Testing/`head -n 1 build/Testing/TAG`/Test.xml .gitlab/ci/ctest-to-junit.xsl apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml

core:docker:
    extends:
        - .docker
        - .testcore
    needs:
        - job: cmp:docker
          artifacts: true
    after_script:
        - .gitlab/ci/transform_ctest_junit.py build/Testing/`head -n 1 build/Testing/TAG`/Test.xml .gitlab/ci/ctest-to-junit.xsl apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml

# CentOS 7
mod:cc7-gcc:
    extends:
        - .centos7
        - .testmod
    needs:
        - job: cmp:cc7-gcc
          artifacts: true

core:cc7-gcc:
    extends:
        - .centos7
        - .testcore
    needs:
        - job: cmp:cc7-gcc
          artifacts: true

# AlmaLinux 9
mod:el9-gcc:
    extends:
        - .alma9
        - .testmod
    needs:
        - job: cmp:el9-gcc
          artifacts: true

core:el9-gcc:
    extends:
        - .alma9
        - .testcore
    needs:
        - job: cmp:el9-gcc
          artifacts: true
          
mod:el9-llvm:
    extends:
        - .alma9
        - .testmod
    variables:
        COMPILER_TYPE: "llvm"
    needs:
        - job: cmp:el9-llvm
          artifacts: true

core:el9-llvm:
    extends:
        - .alma9
        - .testcore
    variables:
        COMPILER_TYPE: "llvm"
    needs:
        - job: cmp:el9-llvm
          artifacts: true


#####################
# Performance tests #
#####################

# Performance tests are only run on Mac OS X and on a native CentOS7 installation. Otherwise we have no control
# about how many CI jobs are performed on the same machine in parallel via the hypervisor.
.testperf:
    stage: performance
    tags:
        - benchmark
    script:
        - ctest -R performance/ --output-on-failure
    allow_failure: true
    retry: 1

# CentOS7
# Deactivated due to lack of runners
.perf:cc7-gcc:
    extends:
        - .centos7
        - .testperf
    needs:
        - job: cmp:cc7-gcc
          artifacts: true


#############################
# Documentation Compilation #
#############################

.doc:
    stage: documentation
    needs: []
    dependencies: []
    tags:
        - k8s-default
    before_script:
        - mkdir public build
        - cd build
    artifacts:
        paths:
            - public
        expire_in: 24 hour

# Compile Doxygen reference
docs:doxygen:
    extends: .doc
    image: $DOCKER_DEPS_IMAGE
    script:
        - curl -sL https://github.com/matusnovak/doxybook2/releases/download/v1.5.0/doxybook2-linux-amd64-v1.5.0.zip --output /tmp/doxybook2-linux64.zip
        - unzip /tmp/doxybook2-linux64.zip -d /usr/local/
        - cmake -DBUILD_DOCS_ONLY=ON ..
        - make apsq_ref_markdown
        - mv doc/reference_markdown ../public/reference_markdown

# Create file tree for hugo
docs:usermanual-hugo:
    extends: .doc
    image: gitlab-registry.cern.ch/sft/docker/alma9:latest
    tags:
        - cvmfs
    script:
        - source ../.ci/init_x86_64.sh
        - cmake -GNinja -DBUILD_DOCS_ONLY=ON ..
        - ninja apsq_docs_hugo
        - mv doc/usermanual_hugo ../public/usermanual_hugo

# Compile LaTeX user manual:
docs:usermanual-pdf:
    extends: .doc
    image: registry.cern.ch/docker.io/thomasweise/docker-texlive-full:latest
    script:
        - apt update && apt install -y git cmake ninja-build python3-yaml pandoc
        - cmake -GNinja -DBUILD_DOCS_ONLY=ON ..
        - ninja apsq_docs_pdf
        - mkdir ../public/usermanual/
        - cp doc/usermanual_pdf/allpix-manual-*.pdf ../public/usermanual/


################################
# Packaging of Binary Tarballs #
################################

.pack:
    stage: packaging
    only:
        - tags@allpix-squared/allpix-squared
        - schedules@allpix-squared/allpix-squared
    before_script:
        - export CCACHE_DIR=`pwd`/ccache
        - source .ci/init_x86_64.sh
        - mkdir -p public/releases
        - cd build
    script:
        - cmake -GNinja -DCMAKE_SKIP_RPATH=ON -DCMAKE_INSTALL_PREFIX=/tmp ..
        - ninja package
        - mv *.tar.gz ../public/releases
    artifacts:
        paths:
            - public
        expire_in: 24 hour

pkg:cc7-gcc:
    extends:
        - .centos7
        - .pack
    needs:
        - job: cmp:cc7-gcc
          artifacts: true

pkg:el9-llvm:
    extends:
        - .alma9
        - .pack
    needs:
        - job: cmp:el9-llvm
          artifacts: true
    variables:
        COMPILER_TYPE: "llvm"

pkg:el9-gcc:
    extends:
        - .alma9
        - .pack
    needs:
        - job: cmp:el9-gcc
          artifacts: true


############################
# Documentation Deployment #
############################

# Automatically deploy documentation to the website
# Deployment job only executed for new tag pushs, not for every commit.
deploy:documentation:
    stage: deployment
    only:
        - tags@allpix-squared/allpix-squared
    trigger:
        project: allpix-squared/allpix-squared-website
        strategy: depend

deploy:eos:
    stage: deployment
    extends:
        - .docker
    variables:
        GIT_STRATEGY: none
    # Only run on new tags for releases and on schedules for nightlies:
    only:
        - tags@allpix-squared/allpix-squared
        - schedules@allpix-squared/allpix-squared
    dependencies:
        - docs:usermanual-pdf
    # Docker image with tools to deploy to EOS
    image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest
    script:
        - if [ ! -z ${CI_COMMIT_TAG} ]; then cp public/usermanual/allpix-manual-*.pdf public/usermanual/allpix-manual.pdf; fi
        - deploy-eos
    # do not run any globally defined before_script or after_script for this step
    before_script: []
    after_script: []

deploy:cvmfs:
    stage: deployment
    dependencies:
        - pkg:cc7-gcc
        - pkg:el9-llvm
        - pkg:el9-gcc
    tags:
        - cvmfs-deploy
    only:
        - tags@allpix-squared/allpix-squared
        - schedules@allpix-squared/allpix-squared
    script:
        - ./.gitlab/ci/download_artifacts.py $API_TOKEN $CI_PROJECT_ID $CI_PIPELINE_ID
        - export RUNNER_LOCATION=$(pwd)
        - if [ -z ${CI_COMMIT_TAG} ]; then export BUILD_PATH='latest'; else export BUILD_PATH=${CI_COMMIT_TAG}; fi
        - sudo -u cvclicdp -i $RUNNER_LOCATION/.gitlab/ci/gitlab_deploy.sh $RUNNER_LOCATION $BUILD_PATH
        - rm -f allpix-squared-*.tar.gz
    retry: 1

.deploy-docker:
    stage: deployment
    extends:
        - .docker
    before_script: []
    image:
        name: gcr.io/kaniko-project/executor:debug
        entrypoint: [""]
    dependencies: []
    script:
        - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
        - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKER_FILE --destination $IMAGE_DESTINATION --build-arg ALLPIX_BUILD_ENV="$ALLPIX_BUILD_ENV"
        - echo "Image pushed successfully to ${IMAGE_DESTINATION}"

deploy:docker-latest:
    extends: .deploy-docker
    variables:
        IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}:latest
        ALLPIX_BUILD_ENV: "CI/Docker"
    only:
        - schedules@allpix-squared/allpix-squared

deploy:docker-tag:
    extends: .deploy-docker
    variables:
        IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}
        ALLPIX_BUILD_ENV: "CI/Docker"
    only:
        - tags@allpix-squared/allpix-squared