Merge branch 'master.dev' into 250-read-in-with-dollar-symbol-not-wor…

…king-with-load-balancing
piclas-framework · May 28, 2024 · c642490 · c642490
2 parents 81ccb45 + 932cc0d
commit c642490
Show file tree

Hide file tree

Showing 497 changed files with 19,438 additions and 3,397 deletions.
diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -126,7 +126,7 @@ if [ -n "$CHANGED" ]; then
 
     # Check if file is executable
     if [ -x "$file" ]; then
-      if [ "$file" != ".githooks/pre-commit" ] && [ "$file" != "docs/documentation/buildPDF.sh" ] && [ "$file" != "docs/documentation/buildHTML.sh" ] ; then
+      if [ "$file" != ".githooks/pre-commit" ] && [ "$file" != "docs/documentation/buildPDF.sh" ] && [ "$file" != "docs/documentation/buildHTML.sh" ] && [ ! -f "tools/Setup_ModuleEnv/$(basename $file)" ] ; then
         # Error header
         ERROR=1
         EXEERR=1

diff --git a/.github/workflows/cmake-ninja.yml b/.github/workflows/cmake-ninja.yml
@@ -318,7 +318,7 @@ jobs:
     - name: Upload artifacts
       uses: actions/upload-artifact@v3
       with:
-        name: piclas-binaries-v3.1.0
+        name: piclas-binaries-v3.2.1
         path: artifacts
 
     - name: Upload release asset

diff --git a/.gitignore b/.gitignore
@@ -84,3 +84,6 @@ unitTests/bin/
 # gitlab-ci-local temp files and folders
 .gitlab-ci-local/
 reggie/
+
+# pyenv project settings
+.python-version
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,3 +1,43 @@
+
+#  # ----------------------------------------------------------------------------------------------------------------------------------------------------
+#  # Use pipeline name to indicate whether MPICH or OpenMPI is used in the complete pipeline execution
+#  # ----------------------------------------------------------------------------------------------------------------------------------------------------
+#  workflow:
+#    name: $PIPELINE_NAME
+#    rules:
+#      - if: '$CI_PIPELINE_CREATED_AT =~ /^\d\d\d\d-\d\d-\d[1,3,5,7,9]T.*/ || $CI_COMMIT_REF_NAME !~ /^gitlab.custom.*/'
+#        variables:
+#          PIPELINE_NAME: '🔵 MPICH: on ${CI_PIPELINE_CREATED_AT} 123 ${CI_COMMIT_REF_NAME} 123'
+#  
+#      - if: '$CI_PIPELINE_CREATED_AT =~ /^\d\d\d\d-\d\d-\d[0,2,4,6,8]T.*/|| $CI_COMMIT_REF_NAME =~ /^gitlab.custom.*/'
+#        variables:
+#          PIPELINE_NAME: '🟠 OpenMPI: $(date +%d) ${CI_COMMIT_REF_NAME}'
+#  
+#      - when: never
+
+# ----------------------------------------------------------------------------------------------------------------------------------------------------
+# Use pipeline name to indicate whether default shared memory, node split or core split is used
+# ----------------------------------------------------------------------------------------------------------------------------------------------------
+workflow:
+  name: $PIPELINE_NAME
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - if: '$DO_NODE_SPLIT'
+      variables:
+        PIPELINE_NAME: '🔵 Node Splitting ${CI_COMMIT_REF_NAME}'
+
+    - if: '$DO_CORE_SPLIT'
+      variables:
+        PIPELINE_NAME: '🟠 Core Splitting ${CI_COMMIT_REF_NAME}'
+
+    - if: '$DO_CORE_SPLIT == null && $DO_NODE_SPLIT == null'
+      variables:
+        PIPELINE_NAME: '🟢 ${CI_COMMIT_REF_NAME}'
+
+    - when: never
+
+# ----------------------------------------------------------------------------------------------------------------------------------------------------
 # Settings
 # ----------------------------------------------------------------------------------------------------------------------------------------------------
 # Load modules on new boltzplatz reggie
@@ -33,6 +73,7 @@ before_script:
       echo $day > reggie/.day;
       ls -la reggie;
       cat reggie/.day;
+      echo "CI_PIPELINE_CREATED_AT $(date -d $CI_PIPELINE_CREATED_AT +%d)";
     fi
   - if [ ! -f "reggie/.day" ]; then
         echo "reggie/.day does not exist! Setting even day number.";
@@ -42,25 +83,26 @@ before_script:
     fi
   - echo "Perform arithmetic with 0-prefixed numbers by telling bash to use base-10 by specifying 10#"
   - if [ "$((10#$day % 2))" == "0" ]; then
-        echo "Even day number ($day). Loading OpenMPI";
-        module purge;
-        module load  cmake/3.26.4    gcc/13.2.0  openmpi/4.1.5/gcc/13.2.0      hdf5/1.14.0/gcc/13.2.0/openmpi/4.1.5      hopr/master/gcc/13.2.0/openmpi/4.1.5/hdf5/1.14.0      petsc/3.19.3/gcc/13.2.0/openmpi/4.1.5;
-      else
-        echo "Odd day number ($day). Loading MPICH";
-        module purge;
-        module load  cmake/3.26.4    gcc/13.2.0  mpich/4.1.2/gcc/13.2.0        hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2        hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0        petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
+      echo "Even day number ($day). Loading OpenMPI";
+      module purge;
+      module load  cmake/3.26.4  gcc/13.2.0  openmpi/4.1.5/gcc/13.2.0      hdf5/1.14.0/gcc/13.2.0/openmpi/4.1.5      hopr/master/gcc/13.2.0/openmpi/4.1.5/hdf5/1.14.0      petsc/3.19.3/gcc/13.2.0/openmpi/4.1.5;
+    else
+      echo "Odd day number ($day). Loading MPICH";
+      module purge;
+      module load  cmake/3.26.4  gcc/13.2.0  mpich/4.1.2/gcc/13.2.0        hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2        hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0        petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
     fi
   - if [ -n "${DO_MPICH}" ]; then
       echo "DO_MPICH activated. Loading MPICH.";
       module purge;
-      module load   cmake/3.26.4    gcc/13.2.0  mpich/4.1.2/gcc/13.2.0        hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2        hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0        petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
+      module load  cmake/3.26.4  gcc/13.2.0  mpich/4.1.2/gcc/13.2.0        hdf5/1.14.0/gcc/13.2.0/mpich/4.1.2        hopr/master/gcc/13.2.0/mpich/4.1.2/hdf5/1.14.0        petsc/3.19.3/gcc/13.2.0/mpich/4.1.2;
     fi
   - if [ -n "${DO_MPICH_DEBUG}" ]; then
       echo "DO_MPICH_DEBUG activated. Loading MPICH with --with-device=ch3:sock which allows over-subscription without performance loss but cannot be used for shared memory arrays safely!";
       module purge;
-      module load   cmake/3.26.4    gcc/13.2.0  mpich-debug/4.1.2/gcc/13.2.0  hdf5/1.14.0/gcc/13.2.0/mpich-debug/4.1.2  hopr/master/gcc/13.2.0/mpich-debug/4.1.2/hdf5/1.14.0  petsc/3.19.3/gcc/13.2.0/mpich-debug/4.1.2;
+      module load  cmake/3.26.4  gcc/13.2.0  mpich-debug/4.1.2/gcc/13.2.0  hdf5/1.14.0/gcc/13.2.0/mpich-debug/4.1.2  hopr/master/gcc/13.2.0/mpich-debug/4.1.2/hdf5/1.14.0  petsc/3.19.3/gcc/13.2.0/mpich-debug/4.1.2;
     fi
   - module list
+
 # ----------------------------------------------------------------------------------------------------------------------------------------------------
 # Stages
 # ----------------------------------------------------------------------------------------------------------------------------------------------------

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,7 @@
 # =========================================================================
 # CMake configuration
 # =========================================================================
+MESSAGE(STATUS "Generating with [cmake] (${CMAKE_VERSION})")
 # NOTE: use minimum CMake version required
 CMAKE_MINIMUM_REQUIRED(VERSION 3.6.3)
 # Policies (see https://cmake.org/cmake/help/v3.13/manual/cmake-policies.7.html)
@@ -63,32 +64,72 @@ INCLUDE(CMakeDependentOption)
 OPTION(PICLAS_PERFORMANCE "Enable performance optimizations (e.g. PGO for GNU)" OFF)
 
 # =========================================================================
-# Check gold support
-# =========================================================================
-EXECUTE_PROCESS(COMMAND ld.gold --version COMMAND grep "^GNU gold" COMMAND sed "s/^.* //g" OUTPUT_VARIABLE GNU_GOLD_VERSION)
-IF (DEFINED GNU_GOLD_VERSION  AND NOT "${GNU_GOLD_VERSION}" STREQUAL "")
-  STRING(STRIP "${GNU_GOLD_VERSION}" GNU_GOLD_VERSION)
-  MESSAGE(STATUS "Setting linker to [gold] (v${GNU_GOLD_VERSION})")
-  # Shift responsibility of driving the final stages of compilation from collect2 to gold via the linker plugin
-  # More information at: https://gcc.gnu.org/wiki/LinkTimeOptimization
-  IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
-    ADD_LINK_OPTIONS("-fuse-ld=gold")
-    # Make it abundantly clear we want to use gold
-    FIND_PROGRAM(CMAKE_GOLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.gold${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
-    SET (CMAKE_LINKER "${CMAKE_GOLD_LINKER}" CACHE FILEPATH "" FORCE)
-    MARK_AS_ADVANCED(FORCE CMAKE_GOLD_LINKER)
+# Check lld / gold support
+# =========================================================================
+# EXECUTE_PROCESS(COMMAND ld.lld --version COMMAND grep "^LLD" COMMAND grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" OUTPUT_VARIABLE LLD_VERSION)
+# # lld should be faster than gold
+# IF (DEFINED LLD_VERSION  AND NOT "${LLD_VERSION}" STREQUAL "")
+#   STRING(STRIP "${LLD_VERSION}" LLD_VERSION)
+#   MESSAGE(STATUS "Setting linker to [lld] (v${LLD_VERSION})")
+#   # Shift responsibility of driving the final stages of compilation from collect2 to gold via the linker plugin
+#   # More information at: https://gcc.gnu.org/wiki/LinkTimeOptimization
+#   IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
+#     ADD_LINK_OPTIONS("-fuse-ld=lld")
+#     # Make it abundantly clear we want to use lld
+#     FIND_PROGRAM(LLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.lld${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+#     SET (CMAKE_LINKER "${LLD_LINKER}" CACHE FILEPATH "" FORCE)
+#     MARK_AS_ADVANCED(FORCE LLD_LINKER)
+#   ELSE()
+#     SET (CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fuse-ld=lld")
+#     SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld")
+#     SET (CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -fuse-ld=lld")
+#     SET (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
+#     # Make it abundantly clear we want to use lld
+#     FIND_PROGRAM(LLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.lld${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+#     SET (CMAKE_LINKER "${LLD_LINKER}" CACHE FILEPATH "" FORCE)
+#     MARK_AS_ADVANCED(FORCE LLD_LINKER)
+#   ENDIF()
+# # gold should be faster than GNU ld
+# ELSE()
+  EXECUTE_PROCESS(COMMAND ld.gold --version COMMAND grep "^GNU gold" COMMAND sed "s/^.* //g" OUTPUT_VARIABLE GNU_GOLD_VERSION)
+  IF (DEFINED GNU_GOLD_VERSION  AND NOT "${GNU_GOLD_VERSION}" STREQUAL "")
+    STRING(STRIP "${GNU_GOLD_VERSION}" GNU_GOLD_VERSION)
+    MESSAGE(STATUS "Linking with [gold] (v${GNU_GOLD_VERSION})")
+    # Shift responsibility of driving the final stages of compilation from collect2 to gold via the linker plugin
+    # More information at: https://gcc.gnu.org/wiki/LinkTimeOptimization
+    IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
+      ADD_LINK_OPTIONS("-fuse-ld=gold")
+      # Make it abundantly clear we want to use gold
+      FIND_PROGRAM(CMAKE_GOLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.gold${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+      SET (CMAKE_LINKER "${CMAKE_GOLD_LINKER}" CACHE FILEPATH "" FORCE)
+      MARK_AS_ADVANCED(FORCE CMAKE_GOLD_LINKER)
+    ELSE()
+      SET (CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fuse-ld=gold")
+      SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
+      # This currently breaks ar (binutils archiver)
+      # SET (CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -fuse-ld=gold")
+      SET (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=gold")
+      # Make it abundantly clear we want to use gold
+      FIND_PROGRAM(CMAKE_GOLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.gold${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+      SET (CMAKE_LINKER "${CMAKE_GOLD_LINKER}" CACHE FILEPATH "" FORCE)
+      MARK_AS_ADVANCED(FORCE CMAKE_GOLD_LINKER)
+    ENDIF()
+# Found neither lld nor gold, output GNU ld
   ELSE()
-    SET (CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    -fuse-ld=gold")
-    SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
-    # This currently breaks ar (binutils archiver)
-    # SET (CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -fuse-ld=gold")
-    SET (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=gold")
-    # Make it abundantly clear we want to use gold
-    FIND_PROGRAM(CMAKE_GOLD_LINKER NAMES ${_CMAKE_TOOLCHAIN_PREFIX}ld.gold${_CMAKE_TOOLCHAIN_SUFFIX} HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
-    SET (CMAKE_LINKER "${CMAKE_GOLD_LINKER}" CACHE FILEPATH "" FORCE)
-    MARK_AS_ADVANCED(FORCE CMAKE_GOLD_LINKER)
+    EXECUTE_PROCESS(COMMAND ld --version OUTPUT_VARIABLE GNU_LD_STRING)
+    # Check if we actually got GNU ld
+    IF (GNU_LD_STRING MATCHES "^GNU ld")
+      EXECUTE_PROCESS(COMMAND ld --version COMMAND grep "^GNU ld" COMMAND sed "s/^.* //g" OUTPUT_VARIABLE GNU_LD_VERSION)
+      STRING(STRIP "${GNU_LD_VERSION}" GNU_LD_VERSION)
+      MESSAGE(STATUS "Linking with [ld] (v${GNU_LD_VERSION})")
+    # ... or if we are running mold
+    ELSEIF (GNU_LD_STRING MATCHES "^mold")
+      EXECUTE_PROCESS(COMMAND ld --version COMMAND grep "^mold" COMMAND grep -Eo "[0-9]+\.[0-9]+\.[0-9]+ " OUTPUT_VARIABLE GNU_MOLD_VERSION)
+      STRING(STRIP "${GNU_MOLD_VERSION}" GNU_MOLD_VERSION)
+      MESSAGE(STATUS "Linking with [mold] (v${GNU_MOLD_VERSION})")
+    ENDIF()
   ENDIF()
-ENDIF()
+# ENDIF()
 
 # =========================================================================
 # code development
@@ -212,6 +253,7 @@ ELSE()
     ENDIF()
   ENDIF()
 ENDIF()
+MESSAGE(STATUS "Compiling with [${BUILD_TYPE_LC}] build type")
 
 # =========================================================================
 # Location of binary and filenames
@@ -248,12 +290,12 @@ endfunction()
 # A function to get all user defined variables with a specified prefix
 SET(configuration ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/configuration.cmake)
 FILE(WRITE ${configuration} "")
-GETLISTOFVARSSTARTINGWITH("PICLAS_" piclasVars)
+GETLISTOFVARSSTARTINGWITH("PICLAS_" piclasVars1)
 GETLISTOFVARSSTARTINGWITH("HDF5_"   piclasVars2)
 GETLISTOFVARSSTARTINGWITH("CMAKE_"  piclasVars3)
 GETLISTOFVARSSTARTINGWITH("POSTI_"  piclasVars4)
 GETLISTOFVARSSTARTINGWITH("LIBS_"   piclasVars5)
-FOREACH (_var IN LISTS piclasVars piclasVars2 piclasVars3 piclasVars4 piclasVars5)
+FOREACH (_var IN LISTS piclasVars1 piclasVars2 piclasVars3 piclasVars4 piclasVars5)
   GET_PROPERTY(currentHelpString CACHE "${_var}" PROPERTY HELPSTRING)
   # Skip empty variables
   IF(NOT ${${_var}} STREQUAL "")