Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Static Builds Against C-Blosc2 #3715

Merged
merged 3 commits into from
Aug 11, 2023
Merged

Fix: Static Builds Against C-Blosc2 #3715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8333d5e
cmake: Add Blosc2 2.10.1 compatibility.
vicentebolea Aug 10, 2023
59fd7dc
CI: added blosc2 dependency to Spack build
vicentebolea Aug 10, 2023
47082ef
CI: use full image tag
vicentebolea Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,8 @@ if(ADIOS2_HAVE_MPI)
add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX)
endif()

cmake_dependent_option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries"
ON "Blosc2_shlib_available" OFF)

#------------------------------------------------------------------------------#
# POSIX O_DIRECT is only working for Unix in adios for now
Expand Down
25 changes: 22 additions & 3 deletions cmake/DetectOptions.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,31 @@ endfunction()

# Blosc2
if(ADIOS2_USE_Blosc2 STREQUAL AUTO)
find_package(Blosc2 2.4)
# Prefect CONFIG mode
find_package(Blosc2 2.4 CONFIG QUIET)
if(NOT Blosc2_FOUND)
find_package(Blosc2 2.4 MODULE QUIET)
endif()
elseif(ADIOS2_USE_Blosc2)
find_package(Blosc2 2.4 REQUIRED)
# Prefect CONFIG mode
find_package(Blosc2 2.4 CONFIG REQUIRED)
if(NOT Blosc2_FOUND)
find_package(Blosc2 2.4 MODULE REQUIRED)
endif()
endif()
if(BLOSC2_FOUND)
if(Blosc2_FOUND)
set(ADIOS2_HAVE_Blosc2 TRUE)
if(TARGET Blosc2::blosc2_shared)
set(Blosc2_shlib_available ON)
endif()

set(adios2_blosc2_tgt Blosc2::Blosc2)
if (Blosc2_VERSION VERSION_GREATER_EQUAL 2.10.1)
set(adios2_blosc2_tgt Blosc2::blosc2_$<IF:$<BOOL:${ADIOS2_Blosc2_PREFER_SHARED}>,shared,static>)
Comment on lines +91 to +92
Copy link
Contributor Author

@ax3l ax3l Aug 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vicentebolea there is a bug here:
ADIOS2_Blosc2_PREFER_SHARED should not assume there is always a shared c-blosc2 library available.

C-Blosc2 builds allow to control static and shared builds independently, thus this will cause issues with ADIOS2_Blosc2_PREFER_SHARED=ON (default) for purely static builds (prefer shared != require shared).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing this out.

ADIOS2_Blosc2_PREFER_SHARED should not assume there is always a shared c-blosc2 library available.

We do not assume this, a couple of lines above we decide this. However, the prefer is not respected since if the user force ADIOS2_Blosc2_PREFER_SHARED=ON it will fail with an static lib

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's what I mean, thx! Do you want to patch this or track it for later patching?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A possible solution at #3767

endif()

add_library(adios2_blosc2 INTERFACE)
target_link_libraries(adios2_blosc2 INTERFACE ${adios2_blosc2_tgt})
endif()

# BZip2
Expand Down
17 changes: 17 additions & 0 deletions scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-base
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,21 @@ RUN apt-get update && apt-get install -y \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \
update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100

# NONSPACK dependency
WORKDIR /opt/blosc2
RUN curl -L https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.1.tar.gz | tar -xvz && \
mkdir build && \
cd build && \
. /spack/share/spack/setup-env.sh && \
spack env activate adios2-ci-serial && \
cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/blosc2/2.10.1 ../c-blosc2-2.10.1 && \
cmake --build . -j$(grep -c '^processor' /proc/cpuinfo) && \
cmake --install . && \
cd .. && \
rm -rf c-blosc2-2.10.1 build

ENV PATH /opt/cmake/bin:/opt/blosc2/2.10.1/bin:${PATH}
ENV LD_LIBRARY_PATH /opt/blosc2/2.10.1/lib64:${LD_LIBRARY_PATH}
ENV CMAKE_PREFIX_PATH /opt/blosc2/2.10.1:${CMAKE_PREFIX_PATH}

ENTRYPOINT ["/bin/bash", "--login"]
2 changes: 1 addition & 1 deletion scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-clang
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM adios2:ci-spack-ubuntu20.04-base
FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
ARG CLANG_VERSION=10

RUN apt-get update && apt-get install -y \
Expand Down
2 changes: 1 addition & 1 deletion scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-gcc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM adios2:ci-spack-ubuntu20.04-base
FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
ARG GCC_VERSION=8

RUN apt-get update && apt-get install -y \
Expand Down
2 changes: 1 addition & 1 deletion scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-intel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM adios2:ci-spack-ubuntu20.04-base
FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base

RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg && \
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
Expand Down
23 changes: 8 additions & 15 deletions scripts/ci/images/build-ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,22 @@
set -ex

# Build the base image
docker build --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base -t adios2:ci-spack-ubuntu20.04-base .
docker build --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base .

# Which is also the gcc11 image
docker tag adios2:ci-spack-ubuntu20.04-base adios2:ci-spack-ubuntu20.04-gcc11
docker tag ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc11

# Build the gcc8, gcc9, and gcc10 images
docker build --rm --build-arg GCC_VERSION=8 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t adios2:ci-spack-ubuntu20.04-gcc8 .
docker build --rm --build-arg GCC_VERSION=9 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t adios2:ci-spack-ubuntu20.04-gcc9 .
docker build --rm --build-arg GCC_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t adios2:ci-spack-ubuntu20.04-gcc10 .
docker build --rm --build-arg GCC_VERSION=8 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8 .
docker build --rm --build-arg GCC_VERSION=9 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc9 .
docker build --rm --build-arg GCC_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc10 .

# Build the clang6 and clang10 images
docker build --rm --build-arg CLANG_VERSION=6.0 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t adios2:ci-spack-ubuntu20.04-clang6 .
docker build --rm --build-arg CLANG_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t adios2:ci-spack-ubuntu20.04-clang10 .

# Tag images for pushing
docker tag adios2:ci-spack-ubuntu20.04-gcc8 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8
docker tag adios2:ci-spack-ubuntu20.04-gcc9 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc9
docker tag adios2:ci-spack-ubuntu20.04-gcc10 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc10
docker tag adios2:ci-spack-ubuntu20.04-gcc11 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc11
docker tag adios2:ci-spack-ubuntu20.04-clang6 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang6
docker tag adios2:ci-spack-ubuntu20.04-clang10 ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang10
docker build --rm --build-arg CLANG_VERSION=6.0 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang6 .
docker build --rm --build-arg CLANG_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang10 .

# Push images to github container registry
docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8
docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc9
docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc10
Expand Down
6 changes: 4 additions & 2 deletions source/adios2/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,11 @@ if(ADIOS2_HAVE_DataSpaces)
target_link_libraries(adios2_core_mpi PRIVATE DataSpaces::DataSpaces)
endif()

set(maybe_adios2_blosc2)
if(ADIOS2_HAVE_Blosc2)
target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp)
target_link_libraries(adios2_core PRIVATE Blosc2::Blosc2)
target_link_libraries(adios2_core PRIVATE adios2_blosc2)
set(maybe_adios2_blosc2 adios2_blosc2)
endif()

if(ADIOS2_HAVE_BZip2)
Expand Down Expand Up @@ -448,7 +450,7 @@ install(DIRECTORY toolkit/
)

# Library installation
install(TARGETS adios2_core ${maybe_adios2_core_mpi} ${maybe_adios2_core_cuda} ${maybe_adios2_core_kokkos} EXPORT adios2Exports
install(TARGETS adios2_core ${maybe_adios2_core_mpi} ${maybe_adios2_core_cuda} ${maybe_adios2_core_kokkos} ${maybe_adios2_blosc2} EXPORT adios2Exports
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_core-runtime
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT adios2_core-libraries NAMELINK_COMPONENT adios2_core-development
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT adios2_core-development
Expand Down