Merge pull request #4269 from vicentebolea/ci-macos-use-conda

ci,macos: use conda for all deps

.github/workflows/everything.yml

@@ -276,31 +276,38 @@ jobs:
 
     runs-on: ${{ matrix.image }}
     env:
-      GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }}
+      # Only way to source a file in a non interactive/non login shell.
+      BASH_ENV: "/Users/runner/.bash_profile"
+      CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
+      CCACHE_COMPRESSLEVEL: 6
+      CCACHE_COMPRESS: true
+      CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
       GH_YML_BASE_OS: macOS
-      GH_YML_MATRIX_OS: ${{ matrix.image }}
+      GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }}
+      GH_YML_MATRIX_ARCH : ${{ matrix.arch }}
       GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }}
+      GH_YML_MATRIX_OS: ${{ matrix.image }}
       GH_YML_MATRIX_PARALLEL: ${{ matrix.parallel }}
-      CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
-      CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
-      CCACHE_COMPRESS: true
-      CCACHE_COMPRESSLEVEL: 6
 
     strategy:
       fail-fast: false
       matrix:
         image: [macos-13, macos-14]
-        shared: [static, shared]
+        shared: [shared, static]
         include:
-          - parallel: serial
           - image: macos-13
+            arch: x64
             compiler: xcode14_2
           - image: macos-14
+            arch: arm64
             compiler: xcode15_4
+          - parallel: serial
         exclude:
           - image: macos-13
             shared: static
 
+    name: macos (${{ matrix.image }}, ${{ matrix.compiler}}, ${{ matrix.arch }}, ${{ matrix.shared }}, ${{ matrix.parallel }})
+
     steps:
       - uses: actions/checkout@v4
         with:

scripts/ci/cmake/ci-macos-13-xcode14_2-shared-serial.cmake

@@ -7,9 +7,21 @@ set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
 ADIOS2_BUILD_EXAMPLES:BOOL=ON
 
+ADIOS2_USE_AWSSDK:BOOL=ON
+ADIOS2_USE_Blosc2:BOOL=ON
+ADIOS2_USE_Bzip2:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
 ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=OFF
-ADISO2_USE_Python:BOOL=ON
+ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_Sodium:BOOL=ON
+ADIOS2_USE_ZeroMQ:BOOL=ON
+ADIOS2_USE_ZFP:BOOL=ON
+
+Python_ROOT_DIR:PATH=$ENV{CMAKE_PREFIX_PATH}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
 
 CMAKE_C_COMPILER_LAUNCHER=ccache
 CMAKE_CXX_COMPILER_LAUNCHER=ccache
@@ -22,7 +34,8 @@ set(ENV{MACOSX_DEPLOYMENT_TARGET} "13.6")
 set(CTEST_CMAKE_GENERATOR "Ninja")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 set(CTEST_TEST_ARGS
-  # Unclear why these tests currently die.  Disabling until it can be addressed.
-  EXCLUDE "Install.Make.*"
+  # Install.Make ticket: https://github.com/ornladios/ADIOS2/issues/4272
+  # DataMan tickets: https://github.com/ornladios/ADIOS2/issues/4273
+  EXCLUDE "Install.Make|Test.Engine.DataManSingleValues|Test.Engine.DataMan1xN.Serial"
 )
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-macos-14-xcode15_4-shared-serial.cmake

@@ -7,9 +7,21 @@ set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
 ADIOS2_BUILD_EXAMPLES:BOOL=ON
 
+ADIOS2_USE_AWSSDK:BOOL=ON
+ADIOS2_USE_Blosc2:BOOL=ON
+ADIOS2_USE_Bzip2:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
 ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_HDF5:BOOL=ON
 ADIOS2_USE_MPI:BOOL=OFF
-ADISO2_USE_Python:BOOL=ON
+ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_Sodium:BOOL=ON
+ADIOS2_USE_ZeroMQ:BOOL=ON
+ADIOS2_USE_ZFP:BOOL=ON
+
+Python_ROOT_DIR:PATH=$ENV{CMAKE_PREFIX_PATH}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
 
 CMAKE_C_COMPILER_LAUNCHER=ccache
 CMAKE_CXX_COMPILER_LAUNCHER=ccache
@@ -22,7 +34,8 @@ set(ENV{MACOSX_DEPLOYMENT_TARGET} "14.5")
 set(CTEST_CMAKE_GENERATOR "Ninja")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 set(CTEST_TEST_ARGS
-  # Unclear why these tests currently die.  Disabling until it can be addressed.
-  EXCLUDE "Install.Make.*"
+  # Install.Make ticket: https://github.com/ornladios/ADIOS2/issues/4272
+  # DataMan tickets: https://github.com/ornladios/ADIOS2/issues/4273
+  EXCLUDE "Install.Make|Test.Engine.DataManSingleValues|Test.Engine.DataMan1xN.Serial"
 )
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-macos-14-xcode15_4-static-serial.cmake

@@ -46,7 +46,8 @@ set(ENV{MACOSX_DEPLOYMENT_TARGET} "14.5")
 set(CTEST_CMAKE_GENERATOR "Ninja")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 set(CTEST_TEST_ARGS
-  # Unclear why these tests currently die.  Disabling until it can be addressed.
-  EXCLUDE "Install.Make.*"
+  # Install.Make ticket: https://github.com/ornladios/ADIOS2/issues/4272
+  # DataMan tickets: https://github.com/ornladios/ADIOS2/issues/4273
+  EXCLUDE "Install.Make|Test.Engine.DataManSingleValues|Test.Engine.DataMan1xN.Serial"
 )
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/gh-actions/conda-env-macos.yml

@@ -0,0 +1,13 @@
+name: adios2
+variables:
+  CMAKE_PREFIX_PATH: "/Users/runner/miniconda3/envs/adios2/:$CMAKE_PREFIX_PATH"
+dependencies:
+  - aws-sdk-cpp
+  - c-blosc2
+  - ccache
+  - hdf5
+  - ninja
+  - numpy>=1.19
+  - python=3.10
+  - zeromq
+  - zfp

scripts/ci/gh-actions/macos-setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -xe
 
 echo "Setting up default XCode version"
 if [ -z "${GH_YML_MATRIX_COMPILER}" ]
@@ -15,40 +15,39 @@ then
   exit 2
 fi
 sudo xcode-select --switch "/Applications/Xcode_${XCODE_VER}.app"
+sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran
 
-echo "Installing CMake"
-
-{
-  readonly version="3.24.2"
-  readonly checksum="efb11a78c064dd7c54a50b8da247254d252112c402c6e48cb7db3f9c84a4e5ad"
-  readonly pkg="cmake-${version}-macos-universal.tar.gz"
-  echo "${checksum}  ${pkg}" > cmake.sha256sum
-
-  curl -OL "https://github.com/Kitware/CMake/releases/download/v${version}/${pkg}"
-  shasum -a 256 --check cmake.sha256sum
-
-  sudo tar -xvzf ${pkg} --strip-components 1 -C /Applications/
-}
-
+echo "Installing Miniconda"
 
-echo "Installing Ninja"
-brew install ninja
+if [ "${RUNNER_ARCH}" = "X64" ]
+then
+  readonly checksum="6d7c1cc138adfc4bb2ccbb8a22eb8e9eb13a366b6af0d63245b643e6c3a3c708"
+  readonly pkg="Miniconda3-py310_24.5.0-0-MacOSX-x86_64.sh"
+elif [ "${RUNNER_ARCH}" = "ARM64" ]
+then
+  readonly checksum="e422602aa19140c600b5889e5b41a0d7187640107ea82fcb5da857dd25330148"
+  readonly pkg="Miniconda3-py310_24.5.0-0-MacOSX-arm64.sh"
+else
+  echo "Error: unknown platform: ${RUNNER_ARCH} "
+  exit 3
+fi
+echo "${checksum}  ${pkg}" > miniconda.sha256sum
 
-echo "Installing GCC"
-brew install gcc
-sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran
+curl -OL "https://repo.anaconda.com/miniconda/${pkg}"
+shasum -a 256 --check miniconda.sha256sum
+bash "./${pkg}" -b
 
-echo "Installing blosc compression"
-brew install c-blosc
+# shellcheck source=/dev/null
+source "/Users/runner/miniconda3/bin/activate"
 
-echo "Installing python3"
-brew install python numpy
+# Canonical installation of Miniconda
+conda init --all
+conda update --all -y
+conda config --add channels conda-forge
+conda config --set channel_priority strict
 
-echo "Installing ccache"
-brew install ccache
+conda env create --verbose -f "gha/scripts/ci/gh-actions/conda-env-macos.yml"
 
-if [[ "$GH_YML_JOBNAME" =~ -ompi ]]
-then
-  echo "Installing OpenMPI"
-  brew install openmpi mpi4py
-fi
+conda list -n adios2
+conda info --verbose
+echo 'conda activate adios2' >> /Users/runner/.bash_profile