From 212182777210574d34412a210efe485e3f5eebb4 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 26 Nov 2024 20:59:07 -0500 Subject: [PATCH] ci: rehaul sanitizer builds - Based them on opensuse tumbleweed to get recent versions of their packages. - Update clang/llvm to 17 - Add build-images-opensuse-tw script --- .gitlab/gitlab-ci-spack.yml | 18 +-- scripts/ci/cmake/ci-common.cmake | 5 + scripts/ci/cmake/ci-el7-spack.cmake | 2 +- scripts/ci/cmake/ci-el7.cmake | 2 +- scripts/ci/cmake/ci-uo-sanitizer-asan.cmake | 11 +- scripts/ci/cmake/ci-uo-sanitizer-msan.cmake | 12 +- scripts/ci/cmake/ci-uo-sanitizer-tsan.cmake | 10 +- scripts/ci/cmake/ci-uo-sanitizer-ubsan.cmake | 12 +- scripts/ci/images/fedora-asan/Dockerfile | 21 ---- .../images/fedora-sanitizers-base/Dockerfile | 22 ---- scripts/ci/images/fedora-tsan/Dockerfile | 10 -- scripts/ci/images/fedora-ubsan/Dockerfile | 8 -- .../opensuse-tw/build-images-opensuse-tw | 49 +++++++++ .../opensuse-tw/opensuse-tw-asan.dockerfile | 20 ++++ ...opensuse-tw-full-stack-onbuild.dockerfile} | 103 +++++++----------- .../opensuse-tw-msan.dockerfile} | 7 +- .../opensuse-tw-sanitizer-base.dockerfile | 33 ++++++ .../opensuse-tw/opensuse-tw-tsan.dockerfile | 9 ++ .../opensuse-tw/opensuse-tw-ubsan.dockerfile | 5 + .../toolchain-msan.cmake} | 4 +- .../toolchain-tsan.cmake} | 0 scripts/dashboard/common.cmake | 2 +- 22 files changed, 218 insertions(+), 147 deletions(-) delete mode 100644 scripts/ci/images/fedora-asan/Dockerfile delete mode 100644 scripts/ci/images/fedora-sanitizers-base/Dockerfile delete mode 100644 scripts/ci/images/fedora-tsan/Dockerfile delete mode 100644 scripts/ci/images/fedora-ubsan/Dockerfile create mode 100755 scripts/ci/images/opensuse-tw/build-images-opensuse-tw create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile rename scripts/ci/images/{fedora-full-stack-onbuild/Dockerfile => opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile} (64%) rename scripts/ci/images/{fedora-msan/Dockerfile => opensuse-tw/opensuse-tw-msan.dockerfile} (57%) create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile rename scripts/ci/images/{fedora-msan/toolchain.cmake => opensuse-tw/toolchain-msan.cmake} (52%) rename scripts/ci/images/{fedora-tsan/toolchain.cmake => opensuse-tw/toolchain-tsan.cmake} (100%) diff --git a/.gitlab/gitlab-ci-spack.yml b/.gitlab/gitlab-ci-spack.yml index 535458f266..ab7f9e408d 100644 --- a/.gitlab/gitlab-ci-spack.yml +++ b/.gitlab/gitlab-ci-spack.yml @@ -1,3 +1,4 @@ +# This file is used to define the GitLab CI/CD pipeline for the ADIOS2 project. default: interruptible: true tags: @@ -35,6 +36,7 @@ default: - $CI_BIN_DIR/testing/ - $CI_BIN_DIR/source/**/*.h - $CI_BIN_DIR/source/**/*.in + - $CI_BIN_DIR/adios2_reorganize_wrapper # CTest and CMake install files. - $CI_BIN_DIR/CMakeCache.txt @@ -54,38 +56,38 @@ default: #=============================================================================== build:uo-sanitizer-asan: - image: ornladios/adios2:ci-fedora-asan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan-20241205 extends: [.common-build] build:uo-sanitizer-msan: - image: ornladios/adios2:ci-fedora-msan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan-20241205 extends: [.common-build] build:uo-sanitizer-tsan: - image: ornladios/adios2:ci-fedora-tsan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan-20241205 extends: [.common-build] build:uo-sanitizer-ubsan: - image: ornladios/adios2:ci-fedora-ubsan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan-20241205 extends: [.common-build] test:uo-sanitizer-asan: - image: ornladios/adios2:ci-fedora-asan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan-20241205 extends: [.common-test] needs: [build:uo-sanitizer-asan] test:uo-sanitizer-msan: - image: ornladios/adios2:ci-fedora-msan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan-20241205 extends: [.common-test] needs: [build:uo-sanitizer-msan] test:uo-sanitizer-tsan: - image: ornladios/adios2:ci-fedora-tsan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan-20241205 extends: [.common-test] needs: [build:uo-sanitizer-tsan] test:uo-sanitizer-ubsan: - image: ornladios/adios2:ci-fedora-ubsan + image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan-20241205 extends: [.common-test] needs: [build:uo-sanitizer-ubsan] diff --git a/scripts/ci/cmake/ci-common.cmake b/scripts/ci/cmake/ci-common.cmake index 033692ad98..5abbba8675 100644 --- a/scripts/ci/cmake/ci-common.cmake +++ b/scripts/ci/cmake/ci-common.cmake @@ -18,10 +18,15 @@ if(NOT CTEST_BUILD_FLAGS) set(CTEST_BUILD_FLAGS "-j${N2CPUS}") endif() endif() + if(NOT PARALLEL_LEVEL IN_LIST CTEST_TEST_ARGS) list(APPEND CTEST_TEST_ARGS PARALLEL_LEVEL ${N2CPUS}) endif() +if(NOT PARALLEL_LEVEL IN_LIST CTEST_MEMCHECK_ARGS) + list(APPEND CTEST_MEMCHECK_ARGS PARALLEL_LEVEL ${N2CPUS}) +endif() + if(NOT dashboard_model) set(dashboard_model Experimental) endif() diff --git a/scripts/ci/cmake/ci-el7-spack.cmake b/scripts/ci/cmake/ci-el7-spack.cmake index 0a0c378c21..f6b19eb4b0 100644 --- a/scripts/ci/cmake/ci-el7-spack.cmake +++ b/scripts/ci/cmake/ci-el7-spack.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com include(ProcessorCount) ProcessorCount(NCPUS) diff --git a/scripts/ci/cmake/ci-el7.cmake b/scripts/ci/cmake/ci-el7.cmake index c11c5f0ec9..f9e5c743c0 100644 --- a/scripts/ci/cmake/ci-el7.cmake +++ b/scripts/ci/cmake/ci-el7.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com set(ENV{CC} gcc) set(ENV{CXX} g++) diff --git a/scripts/ci/cmake/ci-uo-sanitizer-asan.cmake b/scripts/ci/cmake/ci-uo-sanitizer-asan.cmake index cb7ba1fcc2..c56a0e319a 100644 --- a/scripts/ci/cmake/ci-uo-sanitizer-asan.cmake +++ b/scripts/ci/cmake/ci-uo-sanitizer-asan.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com set(ENV{CC} clang) set(ENV{CXX} clang++) @@ -8,6 +8,8 @@ set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/scripts/ci/cmake/adios-a set(ENV{CFLAGS} "${ASAN_FLAGS}") set(ENV{CXXFLAGS} "${ASAN_FLAGS}") +set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "detect_odr_violation=0") + set(dashboard_cache " BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON @@ -23,6 +25,13 @@ set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Ninja") set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") +list(APPEND EXCLUDE_EXPRESSIONS + "Install.*" + "Unit.FileTransport.FailOnEOF.Serial" + ) +list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) +set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") + set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-uo-sanitizer-msan.cmake b/scripts/ci/cmake/ci-uo-sanitizer-msan.cmake index 83ad3f4f77..031a9f712d 100644 --- a/scripts/ci/cmake/ci-uo-sanitizer-msan.cmake +++ b/scripts/ci/cmake/ci-uo-sanitizer-msan.cmake @@ -1,4 +1,7 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) set(dashboard_cache " BUILD_TESTING:BOOL=ON @@ -19,8 +22,11 @@ set(CTEST_CMAKE_GENERATOR "Ninja") set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer") list(APPEND EXCLUDE_EXPRESSIONS - "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP3.Serial" - "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP4.Serial" + "Install.*" + "Staging.1x1DefSync.BP3" + "Engine.DataMan.DataManEngineTest" + "Unit.FileTransport.FailOnEOF.Serial" + "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.*.Serial" ) list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") diff --git a/scripts/ci/cmake/ci-uo-sanitizer-tsan.cmake b/scripts/ci/cmake/ci-uo-sanitizer-tsan.cmake index 43b12a6632..412c62cd66 100644 --- a/scripts/ci/cmake/ci-uo-sanitizer-tsan.cmake +++ b/scripts/ci/cmake/ci-uo-sanitizer-tsan.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com set(dashboard_cache " BUILD_TESTING:BOOL=ON @@ -18,6 +18,14 @@ set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Ninja") set(CTEST_MEMORYCHECK_TYPE "ThreadSanitizer") +list(APPEND EXCLUDE_EXPRESSIONS + "Install.*" + "Unit.FileTransport.FailOnEOF.Serial" + "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.*.Serial" + ) +list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) +set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") + set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-uo-sanitizer-ubsan.cmake b/scripts/ci/cmake/ci-uo-sanitizer-ubsan.cmake index e299f8af73..805ca8f843 100644 --- a/scripts/ci/cmake/ci-uo-sanitizer-ubsan.cmake +++ b/scripts/ci/cmake/ci-uo-sanitizer-ubsan.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com set(ENV{CC} gcc) set(ENV{CXX} g++) @@ -7,7 +7,7 @@ set(UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -pthread") set(ENV{CFLAGS} "${UBSAN_FLAGS}") set(ENV{CXXFLAGS} "${UBSAN_FLAGS}") set(ENV{FFLAGS} "${UBSAN_FLAGS}") -set(ENV{UBSAN_OPTIONS} "print_stacktrace=1") +set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "print_stacktrace=1") set(dashboard_cache " BUILD_TESTING:BOOL=ON @@ -15,6 +15,7 @@ ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_HDF5:STRING=ON ADIOS2_USE_MPI:STRING=OFF +ADIOS2_USE_Python:STRING=OFF HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/usr/bin/h5cc HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/bin/h5diff @@ -24,6 +25,13 @@ set(dashboard_track "Analysis") set(CTEST_CMAKE_GENERATOR "Ninja") set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer") +list(APPEND EXCLUDE_EXPRESSIONS + "Install.*" + "Unit.FileTransport.FailOnEOF.Serial" + ) +list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) +set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") + set(ADIOS_TEST_REPEAT 0) list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/images/fedora-asan/Dockerfile b/scripts/ci/images/fedora-asan/Dockerfile deleted file mode 100644 index e8daae0ca0..0000000000 --- a/scripts/ci/images/fedora-asan/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base - -# Install core dev packages -RUN dnf -y install libasan python3-devel python3-numpy hdf5-devel - -# Install ZFP -WORKDIR /opt/zfp -RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 ../zfp-0.5.5 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf zfp-0.5.5 build -ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/fedora-sanitizers-base/Dockerfile b/scripts/ci/images/fedora-sanitizers-base/Dockerfile deleted file mode 100644 index 7d4948facd..0000000000 --- a/scripts/ci/images/fedora-sanitizers-base/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM fedora:latest - -# Install core dev packages -RUN dnf -y install gcc gcc-c++ gcc-gfortran git make curl file patch \ - zlib-devel bzip2 bzip2-libs bzip2-devel blosc-devel vim valgrind \ - zeromq-devel libpng-devel libfabric-devel libffi-devel \ - python3-devel python3-numpy \ - clang llvm-devel libcxxabi-devel libcxx-devel - -# Install the most recent CMake nightly binary -WORKDIR /opt/cmake -RUN curl -L https://cmake.org/files/dev/$(curl https://cmake.org/files/dev/ | sed -n '/Linux-x86_64.tar.gz/s/.*>\(cmake[^<]*\)<.*/\1/p' | sort | tail -1) | tar --strip-components=1 -xzv -ENV PATH=/opt/cmake/bin:${PATH} - -# Installing Kitware Ninja -RUN mkdir -p /usr/local/bin && \ - curl -L https://github.com/Kitware/ninja/releases/download/v1.9.0.g99df1.kitware.dyndep-1.jobserver-1/ninja-1.9.0.g99df1.kitware.dyndep-1.jobserver-1_x86_64-linux-gnu.tar.gz | \ - tar -C /usr/local/bin --strip-components=1 -xzv - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/fedora-tsan/Dockerfile b/scripts/ci/images/fedora-tsan/Dockerfile deleted file mode 100644 index 123daf1bab..0000000000 --- a/scripts/ci/images/fedora-tsan/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG INSTALL_PREFIX=/opt/tsan -ARG TOOLCHAIN_FILE=toolchain.cmake -ARG CFLAGS="-fsanitize=thread" -ARG CXXFLAGS="-fsanitize=thread" -ARG LLVM_USE_SANITIZER=Thread - -FROM ornladios/adios2:ci-fedora-full-stack-onbuild AS tmp-stage - -FROM ornladios/adios2:ci-fedora-sanitizers-base -COPY --from=tmp-stage /opt/tsan/ /opt/tsan/ diff --git a/scripts/ci/images/fedora-ubsan/Dockerfile b/scripts/ci/images/fedora-ubsan/Dockerfile deleted file mode 100644 index 1af105610f..0000000000 --- a/scripts/ci/images/fedora-ubsan/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base - -# Install core dev packages -RUN dnf -y install libubsan python3-devel python3-numpy hdf5-devel - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/opensuse-tw/build-images-opensuse-tw b/scripts/ci/images/opensuse-tw/build-images-opensuse-tw new file mode 100755 index 0000000000..26c15b2d53 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/build-images-opensuse-tw @@ -0,0 +1,49 @@ +#!/bin/bash + +# This script is used to build the openSUSE Tumbleweed images + +set -xeuo pipefail + +declare -r DOCKER_REPOSITORY=ghcr.io/ornladios/adios2 +declare -u ENABLE_PUSH= + +#============================================================================= +# Functions + +# Function to build a docker image +function docker_build() { + local docker_file=$1 + local docker_tag=$2 + local args=(--progress=plain --rm) + + if [[ -n "${ENABLE_PUSH}" ]]; then + args+=("--push") + fi + + sudo docker build "${args[@]}" \ + --file "${docker_file}" \ + --tag "${DOCKER_REPOSITORY}:${docker_tag}-$(date +"%Y%m%d")" \ + . +} + +function usage() { + echo "Usage: $0 [-p ] [-m ]" + echo " -p Push to the docker registry" + exit 0 +} + +#============================================================================= +# Main +while getopts "p" o; do + case "${o}" in + p) ENABLE_PUSH=true ;; + *) usage ;; + esac +done + +docker_build opensuse-tw-sanitizer-base.dockerfile ci-opensuse-tw-sanitizer-base +docker_build opensuse-tw-full-stack-onbuild.dockerfile ci-opensuse-tw-full-stack-onbuild +docker_build opensuse-tw-asan.dockerfile ci-opensuse-tw-asan +docker_build opensuse-tw-msan.dockerfile ci-opensuse-tw-msan +docker_build opensuse-tw-tsan.dockerfile ci-opensuse-tw-tsan +docker_build opensuse-tw-ubsan.dockerfile ci-opensuse-tw-ubsan diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile new file mode 100644 index 0000000000..8ceaf63b39 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile @@ -0,0 +1,20 @@ +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base + +# Install core dev packages +RUN zypper install -y --no-recommends libasan8 hdf5-devel && \ + zypper clean --all + +# Install ZFP +WORKDIR /opt/zfp +RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ + cmake -GNinja -S zfp-0.5.5 -B build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \ + && \ + cmake --build build && \ + cmake --install build && \ + rm -rf zfp-0.5.5 build +ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} diff --git a/scripts/ci/images/fedora-full-stack-onbuild/Dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile similarity index 64% rename from scripts/ci/images/fedora-full-stack-onbuild/Dockerfile rename to scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile index 51edf7e6f9..1af5dee0ca 100644 --- a/scripts/ci/images/fedora-full-stack-onbuild/Dockerfile +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile @@ -1,4 +1,4 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base # Set up some arguments ONBUILD ARG INSTALL_PREFIX @@ -7,24 +7,20 @@ ONBUILD ARG CFLAGS ONBUILD ARG CXXFLAGS ONBUILD ARG LLVM_USE_SANITIZER +ENV CC=clang +ENV CXX=clang++ + # Build and install libc++ -# Have to build in debug mode due to a bug in LLVM: -# https://bugs.llvm.org/show_bug.cgi?id=44586 ONBUILD WORKDIR /root/llvm -ONBUILD RUN git clone --branch llvmorg-9.0.0 --depth 1 \ - https://github.com/llvm/llvm-project.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_C_COMPILER=/usr/bin/clang \ - -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ - -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \ - -DLLVM_USE_SANITIZER=${LLVM_USE_SANITIZER} \ - ../source/llvm && \ - ninja install-cxxabi install-cxx +ONBUILD RUN git clone --branch llvmorg-17.0.6 --depth 1 \ + https://github.com/llvm/llvm-project.git source +ONBUILD RUN cmake -GNinja -B build -S source/runtimes \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLLVM_USE_SANITIZER=${LLVM_USE_SANITIZER} \ + && \ + cmake --build build --target install-cxxabi install-cxx # Copy in the toolchain ONBUILD COPY \ @@ -35,19 +31,17 @@ ONBUILD COPY \ ONBUILD WORKDIR /root/zlib ONBUILD RUN git clone --branch v1.2.11 --depth 1 \ https://github.com/madler/zlib.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install bzip2 ONBUILD WORKDIR /root/bzip2 -ONBUILD RUN git clone --branch bzip2-1.0.8 \ +ONBUILD RUN git clone --branch bzip2-1.0.8 --depth 1 \ https://sourceware.org/git/bzip2.git source && \ cd source && \ sed -e "s_^CC=.*\$_CC=/usr/bin/clang_" \ @@ -65,69 +59,54 @@ ONBUILD RUN git clone --branch bzip2-1.0.8 \ # Build and install zeromq ONBUILD WORKDIR /root/zeromq -ONBUILD RUN git clone --branch v4.3.2 \ +ONBUILD RUN git clone --branch v4.3.2 --depth 1 \ https://github.com/zeromq/libzmq.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build + # Build and install libpng ONBUILD WORKDIR /root/libpng -ONBUILD RUN git clone --branch v1.6.9 \ +ONBUILD RUN git clone --branch v1.6.9 --depth 1 \ https://git.code.sf.net/p/libpng/code.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install hdf5 ONBUILD WORKDIR /root/hdf5 -ONBUILD RUN git clone --branch hdf5-1_10_6 \ - https://bitbucket.hdfgroup.org/scm/hdffv/hdf5.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ +ONBUILD RUN git clone --branch hdf5-1_14_3 --depth=1 \ + https://github.com/HDFGroup/hdf5.git source && \ + cmake -GNinja -S source -B build \ + -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install - -# Build and install blosc -ONBUILD WORKDIR /root/blosc -ONBUILD RUN git clone --branch v1.17.1 \ - https://github.com/Blosc/c-blosc.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install libfabric ONBUILD WORKDIR /root/libfabric ONBUILD RUN mkdir -p source build && \ curl -L \ - https://github.com/ofiwg/libfabric/releases/download/v1.9.0/libfabric-1.9.0.tar.bz2 | \ + https://github.com/ofiwg/libfabric/releases/download/v1.18.1/libfabric-1.18.1.tar.bz2 | \ tar -C source -xj --strip-components=1 && \ cd build && \ ../source/configure \ --prefix=${INSTALL_PREFIX} \ + --disable-efa \ + --disable-shm \ + --disable-verbs \ CC="/usr/bin/clang ${CFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument" \ CXX="/usr/bin/clang++ ${CXXFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument -nostdinc++ -isystem ${INSTALL_PREFIX}/include/c++/v1 -stdlib=libc++" && \ make -j4 install @@ -143,4 +122,4 @@ ONBUILD RUN mkdir -p source build && \ --prefix=${INSTALL_PREFIX} \ CC="/usr/bin/clang ${CFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument" \ CXX="/usr/bin/clang++ ${CXXFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument -nostdinc++ -isystem ${INSTALL_PREFIX}/include/c++/v1 -stdlib=libc++" && \ - make -j4 install + make -j4 install diff --git a/scripts/ci/images/fedora-msan/Dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile similarity index 57% rename from scripts/ci/images/fedora-msan/Dockerfile rename to scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile index b289d1f4d6..e922d298f4 100644 --- a/scripts/ci/images/fedora-msan/Dockerfile +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile @@ -1,10 +1,9 @@ ARG INSTALL_PREFIX=/opt/msan -ARG TOOLCHAIN_FILE=toolchain.cmake +ARG TOOLCHAIN_FILE=toolchain-msan.cmake ARG CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins" ARG CXXFLAGS="-fsanitize=memory -fsanitize-memory-track-origins" ARG LLVM_USE_SANITIZER=MemoryWithOrigins -FROM ornladios/adios2:ci-fedora-full-stack-onbuild AS tmp-stage - -FROM ornladios/adios2:ci-fedora-sanitizers-base +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-full-stack-onbuild AS tmp-stage +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base COPY --from=tmp-stage /opt/msan/ /opt/msan/ diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile new file mode 100644 index 0000000000..081d804c36 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile @@ -0,0 +1,33 @@ +FROM opensuse/tumbleweed +LABEL maintainer "Vicente Adolfo Bolea Sanchez" + +# Base dependencies for building ADIOS2 projects +RUN zypper refresh && \ + zypper update -y && \ + zypper install -y --no-recommends \ + awk \ + bzip2 \ + gzip \ + bzip3-devel \ + clang17 \ + cmake \ + curl \ + file \ + gcc \ + gcc-c++ \ + git \ + git-lfs \ + hdf5-devel \ + libffi-devel \ + libpng16-devel \ + libunwind-devel \ + llvm17 \ + ninja \ + patch \ + python3-devel \ + python3-numpy \ + tar \ + zeromq-devel \ + zlib-devel \ + && \ + zypper clean --all diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile new file mode 100644 index 0000000000..295d1ac155 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile @@ -0,0 +1,9 @@ +ARG INSTALL_PREFIX=/opt/tsan +ARG TOOLCHAIN_FILE=toolchain-tsan.cmake +ARG CFLAGS="-fsanitize=thread" +ARG CXXFLAGS="-fsanitize=thread" +ARG LLVM_USE_SANITIZER=Thread + +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-full-stack-onbuild AS tmp-stage +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base +COPY --from=tmp-stage /opt/tsan/ /opt/tsan/ diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile new file mode 100644 index 0000000000..35be3e7f31 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile @@ -0,0 +1,5 @@ +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base + +# Install core dev packages +RUN zypper install -y --no-recommends libubsan1 libasan8 hdf5-devel zfp-devel && \ + zypper clean --all diff --git a/scripts/ci/images/fedora-msan/toolchain.cmake b/scripts/ci/images/opensuse-tw/toolchain-msan.cmake similarity index 52% rename from scripts/ci/images/fedora-msan/toolchain.cmake rename to scripts/ci/images/opensuse-tw/toolchain-msan.cmake index 1015ff783d..724307c620 100644 --- a/scripts/ci/images/fedora-msan/toolchain.cmake +++ b/scripts/ci/images/opensuse-tw/toolchain-msan.cmake @@ -1,7 +1,7 @@ set(CMAKE_C_COMPILER /usr/bin/clang) set(CMAKE_CXX_COMPILER /usr/bin/clang++) -set(_link_flags "-stdlib=libc++ -L /opt/msan/lib -Wl,-rpath,/opt/msan/lib -Wno-unused-command-line-argument") -set(_compile_flags "-nostdinc++ -isystem /opt/msan/include/c++/v1 -fsanitize=memory -fsanitize-memory-track-origins") +set(_link_flags "-stdlib=libc++ -L /opt/msan/lib -lc++abi -Wl,-rpath,/opt/msan/lib -Wno-unused-command-line-argument") +set(_compile_flags "-nostdinc++ -isystem /opt/msan/include -isystem /opt/msan/include/c++/v1 -fsanitize=memory -fsanitize-memory-track-origins") set(CMAKE_EXE_LINKER_FLAGS_INIT ${_link_flags}) set(CMAKE_SHARED_LINKER_FLAGS_INIT ${_link_flags}) set(CMAKE_C_FLAGS_INIT ${_compile_flags}) diff --git a/scripts/ci/images/fedora-tsan/toolchain.cmake b/scripts/ci/images/opensuse-tw/toolchain-tsan.cmake similarity index 100% rename from scripts/ci/images/fedora-tsan/toolchain.cmake rename to scripts/ci/images/opensuse-tw/toolchain-tsan.cmake diff --git a/scripts/dashboard/common.cmake b/scripts/dashboard/common.cmake index 265b5c89e9..df71588478 100644 --- a/scripts/dashboard/common.cmake +++ b/scripts/dashboard/common.cmake @@ -65,7 +65,7 @@ # set(ENV{FC} /path/to/fc) # Fortran compiler (optional) # set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) -cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) if(NOT DEFINED dashboard_full) set(dashboard_full TRUE)